From ee6279d433efd1f6ca4de10a0de3d7a41fc9d5c0 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Mon, 15 Jan 2024 20:12:16 +0800 Subject: [PATCH 01/13] Added --- scormxblock/scormxblock.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index 4a4865b..0bddd08 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -3,10 +3,13 @@ import io import os +from os.path import exists as path_exists +from os import remove as remove_file import pkg_resources import uuid import logging import re +from shutil import rmtree as remove_folder from collections import namedtuple from lxml import etree from urlparse import urlparse @@ -275,6 +278,28 @@ class ScormXBlock(StudioEditableXBlockMixin, ScorableXBlockMixin, XBlock): ) has_author_view = True + def discard_scorm_package(self): + """Remove old scorm package""" + try: + if isinstance(self.fs, OSFS): + _pkg_uuid = self.scorm_pkg.split('/')[0] + if _pkg_uuid: + _pkg_folder_path = self.fs.getsyspath(_pkg_uuid) + if path_exists(_pkg_folder_path): + remove_folder(_pkg_folder_path) + logger.info('[INFO] Old SCORM Package Folder {} removed.'.format(_pkg_folder_path)) + _zip_scorm_package = _pkg_folder_path + '.zip' + if path_exists(_zip_scorm_package): + remove_file(_zip_scorm_package) + logger.info('[INFO] Old SCORM zip package {} removed.'.format(_zip_scorm_package)) + else: + logger.warn('[WARN] Old SCORM Package Folder {} not found.'.format(_pkg_folder_path)) + else: + pass + except Exception as e: + logger.error('[ERROR] Got exception while removing folder {} ---> {}'.format(_pkg_folder_path, str(e))) + raise + # region Studio handler @XBlock.handler def studio_upload_files(self, request, suffix=''): @@ -283,6 +308,8 @@ def studio_upload_files(self, request, suffix=''): cover_images = request._request.FILES.getlist('cover_images[]') if pkg: + self.discard_scorm_package() + with ZipFile(pkg.file, 'r') as zip_fs: mf = zip_fs.read('imsmanifest.xml') self.scorm_pkg_version, scorm_index, scorm_launch = self._get_scorm_info(mf) From af032791d8951900020d0fea6bbb80f7925b6997 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Tue, 16 Jan 2024 11:07:09 +0800 Subject: [PATCH 02/13] added --- scormxblock/scormxblock.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index 0bddd08..fbd5c8d 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -278,22 +278,39 @@ class ScormXBlock(StudioEditableXBlockMixin, ScorableXBlockMixin, XBlock): ) has_author_view = True - def discard_scorm_package(self): - """Remove old scorm package""" + def discard_scorm_package(self, remove_dfs_scorm_folder=False): + """Remove old scorm package + + 1) When admin uploading a new scorm package, we specify this argument `remove_dfs_scorm_folder` with value `False`. + Then the folder `.../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/` will be cleared. + + 2) When admin remove scorm from `Unit`, we specify this argument `remove_dfs_scorm_folder` with value `True`. + Then the folder `.../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65` will be removed. + + """ try: if isinstance(self.fs, OSFS): _pkg_uuid = self.scorm_pkg.split('/')[0] if _pkg_uuid: + # Remove : .../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/2b1fe852ed9c4f59b1be5b7636be9f32 _pkg_folder_path = self.fs.getsyspath(_pkg_uuid) + if remove_dfs_scorm_folder: + _pkg_folder_path = _pkg_folder_path[:_pkg_folder_path.find('/fs/')] + if path_exists(_pkg_folder_path): remove_folder(_pkg_folder_path) logger.info('[INFO] Old SCORM Package Folder {} removed.'.format(_pkg_folder_path)) + + # Remove : .../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/2b1fe852ed9c4f59b1be5b7636be9f32.zip _zip_scorm_package = _pkg_folder_path + '.zip' if path_exists(_zip_scorm_package): remove_file(_zip_scorm_package) logger.info('[INFO] Old SCORM zip package {} removed.'.format(_zip_scorm_package)) + else: logger.warn('[WARN] Old SCORM Package Folder {} not found.'.format(_pkg_folder_path)) + else: + logger.info('[WARN] uuid not found in path {}.'.format(self.scorm_pkg)) else: pass except Exception as e: From 643614572282a404fc93dce43f0619b4cf347074 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Tue, 16 Jan 2024 13:02:49 +0800 Subject: [PATCH 03/13] fixed: deleting from dfs --- scormxblock/scormxblock.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index fbd5c8d..4615bdd 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -278,6 +278,11 @@ class ScormXBlock(StudioEditableXBlockMixin, ScorableXBlockMixin, XBlock): ) has_author_view = True + def release_all_external_resources(self): + """Called when course/chapter/subsection/unit got removed. + """ + self.discard_scorm_package(remove_dfs_scorm_folder=True) + def discard_scorm_package(self, remove_dfs_scorm_folder=False): """Remove old scorm package From b0e2608560aad24b8bbb5c353304e2ec54f670f5 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Tue, 16 Jan 2024 15:00:17 +0800 Subject: [PATCH 04/13] added: for aws s3 --- scormxblock/scormxblock.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index 4615bdd..9e1e4f9 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -316,11 +316,26 @@ def discard_scorm_package(self, remove_dfs_scorm_folder=False): logger.warn('[WARN] Old SCORM Package Folder {} not found.'.format(_pkg_folder_path)) else: logger.info('[WARN] uuid not found in path {}.'.format(self.scorm_pkg)) + else: - pass + S3_BUCKET_NAME = settings.DJFS.get('bucket') + _s3_prefix = self.fs.dir_path[1:] # Sample: /xblock/block--v1-_beta-.Content__demo-.2020Q1-.type_64_scormxblock-.block_64_ae63e8b39db84405a8763c9a5441f93c/fs/NONE.NONE + _pkg_uuid = self.scorm_pkg.split('/')[0] + _delete_keys = {'Objects': []} + objects_to_delete = self.fs.client.list_objects_v2(Bucket=S3_BUCKET_NAME, Prefix=_s3_prefix) + _delete_keys['Objects'] = [ + {'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])] + ] + + s3_resp = self.fs.client.delete_objects(Bucket="MyBucket", Delete=_delete_keys) + _errors = s3_resp.get('Errors', None) + if _errors: + raise Exception(_errors) + + logger.info('[INFO] {} Old SCORM Packages removed from AWS S3.'.format(len(objects_to_delete.get('Contents', [])))) + except Exception as e: - logger.error('[ERROR] Got exception while removing folder {} ---> {}'.format(_pkg_folder_path, str(e))) - raise + logger.error('[ERROR] Got exception while removing package: {}'.format(str(e))) # region Studio handler @XBlock.handler From 470cc76cfaa4f2df4eed75b56ef13bbfd14c16c3 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Tue, 16 Jan 2024 15:09:31 +0800 Subject: [PATCH 05/13] fixed --- scormxblock/scormxblock.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index 9e1e4f9..bbc1f6d 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -318,9 +318,10 @@ def discard_scorm_package(self, remove_dfs_scorm_folder=False): logger.info('[WARN] uuid not found in path {}.'.format(self.scorm_pkg)) else: - S3_BUCKET_NAME = settings.DJFS.get('bucket') _s3_prefix = self.fs.dir_path[1:] # Sample: /xblock/block--v1-_beta-.Content__demo-.2020Q1-.type_64_scormxblock-.block_64_ae63e8b39db84405a8763c9a5441f93c/fs/NONE.NONE - _pkg_uuid = self.scorm_pkg.split('/')[0] + logger.info('[INFO] Removing AWS S3 Old SCORM Packages by PREFIX={}...'.format(_s3_prefix)) + + S3_BUCKET_NAME = settings.DJFS.get('bucket') _delete_keys = {'Objects': []} objects_to_delete = self.fs.client.list_objects_v2(Bucket=S3_BUCKET_NAME, Prefix=_s3_prefix) _delete_keys['Objects'] = [ From 87491566d6cc3097faa2c948ea8f28654beef6b0 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Tue, 16 Jan 2024 16:04:28 +0800 Subject: [PATCH 06/13] log --- scormxblock/scormxblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index bbc1f6d..e041a4e 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -318,10 +318,10 @@ def discard_scorm_package(self, remove_dfs_scorm_folder=False): logger.info('[WARN] uuid not found in path {}.'.format(self.scorm_pkg)) else: + S3_BUCKET_NAME = settings.DJFS.get('bucket') _s3_prefix = self.fs.dir_path[1:] # Sample: /xblock/block--v1-_beta-.Content__demo-.2020Q1-.type_64_scormxblock-.block_64_ae63e8b39db84405a8763c9a5441f93c/fs/NONE.NONE - logger.info('[INFO] Removing AWS S3 Old SCORM Packages by PREFIX={}...'.format(_s3_prefix)) + logger.info('[INFO] Removing AWS S3 Old SCORM Packages by BucketName={}, PREFIX={}...'.format(S3_BUCKET_NAME, _s3_prefix)) - S3_BUCKET_NAME = settings.DJFS.get('bucket') _delete_keys = {'Objects': []} objects_to_delete = self.fs.client.list_objects_v2(Bucket=S3_BUCKET_NAME, Prefix=_s3_prefix) _delete_keys['Objects'] = [ From 7fe28965080deabe684571545f6c5a07e7f8bc1d Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Tue, 16 Jan 2024 16:15:01 +0800 Subject: [PATCH 07/13] fixed: bucket --- scormxblock/scormxblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index e041a4e..3032610 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -328,7 +328,7 @@ def discard_scorm_package(self, remove_dfs_scorm_folder=False): {'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])] ] - s3_resp = self.fs.client.delete_objects(Bucket="MyBucket", Delete=_delete_keys) + s3_resp = self.fs.client.delete_objects(Bucket=S3_BUCKET_NAME, Delete=_delete_keys) _errors = s3_resp.get('Errors', None) if _errors: raise Exception(_errors) From 8ca2b171f34cd2ff0b209be3c6b03d19804553f2 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Tue, 16 Jan 2024 18:06:34 +0800 Subject: [PATCH 08/13] fixed --- scormxblock/scormxblock.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index 3032610..35a9a54 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -281,15 +281,15 @@ class ScormXBlock(StudioEditableXBlockMixin, ScorableXBlockMixin, XBlock): def release_all_external_resources(self): """Called when course/chapter/subsection/unit got removed. """ - self.discard_scorm_package(remove_dfs_scorm_folder=True) + self.discard_scorm_package(remove_scorm_pkg_root=True) - def discard_scorm_package(self, remove_dfs_scorm_folder=False): + def discard_scorm_package(self, remove_scorm_pkg_root=False): """Remove old scorm package - 1) When admin uploading a new scorm package, we specify this argument `remove_dfs_scorm_folder` with value `False`. - Then the folder `.../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/` will be cleared. + 1) When admin uploading a new scorm package, we specify this argument `remove_scorm_pkg_root` with value `False`. + Then the folder `.../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/xxxx_uuid_xxxxx` will be cleared. - 2) When admin remove scorm from `Unit`, we specify this argument `remove_dfs_scorm_folder` with value `True`. + 2) When admin remove scorm from `Unit`, we specify this argument `remove_scorm_pkg_root` with value `True`. Then the folder `.../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65` will be removed. """ @@ -299,7 +299,7 @@ def discard_scorm_package(self, remove_dfs_scorm_folder=False): if _pkg_uuid: # Remove : .../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/2b1fe852ed9c4f59b1be5b7636be9f32 _pkg_folder_path = self.fs.getsyspath(_pkg_uuid) - if remove_dfs_scorm_folder: + if remove_scorm_pkg_root: _pkg_folder_path = _pkg_folder_path[:_pkg_folder_path.find('/fs/')] if path_exists(_pkg_folder_path): @@ -320,6 +320,8 @@ def discard_scorm_package(self, remove_dfs_scorm_folder=False): else: S3_BUCKET_NAME = settings.DJFS.get('bucket') _s3_prefix = self.fs.dir_path[1:] # Sample: /xblock/block--v1-_beta-.Content__demo-.2020Q1-.type_64_scormxblock-.block_64_ae63e8b39db84405a8763c9a5441f93c/fs/NONE.NONE + _pkg_uuid = self.scorm_pkg.split('/')[0] + _s3_prefix = _s3_prefix if remove_scorm_pkg_root else _s3_prefix + '/' + _pkg_uuid logger.info('[INFO] Removing AWS S3 Old SCORM Packages by BucketName={}, PREFIX={}...'.format(S3_BUCKET_NAME, _s3_prefix)) _delete_keys = {'Objects': []} From 89febec33f496c56951248838a1573c91a54d1b8 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Tue, 16 Jan 2024 18:21:08 +0800 Subject: [PATCH 09/13] fixed --- scormxblock/scormxblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index 35a9a54..4267ee0 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -321,7 +321,7 @@ def discard_scorm_package(self, remove_scorm_pkg_root=False): S3_BUCKET_NAME = settings.DJFS.get('bucket') _s3_prefix = self.fs.dir_path[1:] # Sample: /xblock/block--v1-_beta-.Content__demo-.2020Q1-.type_64_scormxblock-.block_64_ae63e8b39db84405a8763c9a5441f93c/fs/NONE.NONE _pkg_uuid = self.scorm_pkg.split('/')[0] - _s3_prefix = _s3_prefix if remove_scorm_pkg_root else _s3_prefix + '/' + _pkg_uuid + _s3_prefix = _s3_prefix if remove_scorm_pkg_root else (_s3_prefix + '/' + _pkg_uuid) logger.info('[INFO] Removing AWS S3 Old SCORM Packages by BucketName={}, PREFIX={}...'.format(S3_BUCKET_NAME, _s3_prefix)) _delete_keys = {'Objects': []} From 7346342d14139fd951e09371a5484a172a09396c Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Wed, 17 Jan 2024 15:51:01 +0800 Subject: [PATCH 10/13] fixed: batch deleting support --- scormxblock/scormxblock.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index 4267ee0..b188fd6 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -318,24 +318,34 @@ def discard_scorm_package(self, remove_scorm_pkg_root=False): logger.info('[WARN] uuid not found in path {}.'.format(self.scorm_pkg)) else: + deleted_count = 0 S3_BUCKET_NAME = settings.DJFS.get('bucket') _s3_prefix = self.fs.dir_path[1:] # Sample: /xblock/block--v1-_beta-.Content__demo-.2020Q1-.type_64_scormxblock-.block_64_ae63e8b39db84405a8763c9a5441f93c/fs/NONE.NONE _pkg_uuid = self.scorm_pkg.split('/')[0] _s3_prefix = _s3_prefix if remove_scorm_pkg_root else (_s3_prefix + '/' + _pkg_uuid) + _kwargs = {'Bucket': S3_BUCKET_NAME, 'Prefix': _s3_prefix} logger.info('[INFO] Removing AWS S3 Old SCORM Packages by BucketName={}, PREFIX={}...'.format(S3_BUCKET_NAME, _s3_prefix)) - _delete_keys = {'Objects': []} - objects_to_delete = self.fs.client.list_objects_v2(Bucket=S3_BUCKET_NAME, Prefix=_s3_prefix) - _delete_keys['Objects'] = [ - {'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])] - ] - - s3_resp = self.fs.client.delete_objects(Bucket=S3_BUCKET_NAME, Delete=_delete_keys) - _errors = s3_resp.get('Errors', None) - if _errors: - raise Exception(_errors) - - logger.info('[INFO] {} Old SCORM Packages removed from AWS S3.'.format(len(objects_to_delete.get('Contents', [])))) + while True: + resp = self.fs.client.list_objects_v2(**_kwargs) + _delete_keys = { + 'Objects': [ + {'Key': k if isinstance(k, unicode) else k.decode('utf-8')} for k in [obj['Key'] for obj in resp.get('Contents', [])] + ] + } + s3_resp = self.fs.client.delete_objects(Bucket=S3_BUCKET_NAME, Delete=_delete_keys) + _errors = s3_resp.get('Errors', None) + if _errors: + raise Exception(_errors) + + deleted_count += len(_delete_keys['Objects']) + + try: + _kwargs['ContinuationToken'] = resp['NextContinuationToken'] + except KeyError: + break + + logger.info('[INFO] {} Old SCORM Packages removed from AWS S3.'.format(deleted_count)) except Exception as e: logger.error('[ERROR] Got exception while removing package: {}'.format(str(e))) From b05ec06ef094dcb7619ddf01f3e97d98fb6ccce0 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Mon, 22 Jan 2024 17:00:23 +0800 Subject: [PATCH 11/13] fixed --- scormxblock/scormxblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index b188fd6..039b27d 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -358,8 +358,6 @@ def studio_upload_files(self, request, suffix=''): cover_images = request._request.FILES.getlist('cover_images[]') if pkg: - self.discard_scorm_package() - with ZipFile(pkg.file, 'r') as zip_fs: mf = zip_fs.read('imsmanifest.xml') self.scorm_pkg_version, scorm_index, scorm_launch = self._get_scorm_info(mf) @@ -376,6 +374,8 @@ def studio_upload_files(self, request, suffix=''): self._upload_scorm_zip(pkg.file, pkg_id) self.scorm_pkg_filename = pkg.filename + self.discard_scorm_package() + if cover_images: self.cover_images = [ self._upload_cover_image(c) for c in cover_images From ddde38d3c770e582f09fb1f027a7c7847feb2e77 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Fri, 26 Jan 2024 14:52:52 +0800 Subject: [PATCH 12/13] Fixed: removement of expired packages --- scormxblock/scormxblock.py | 48 ++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index 039b27d..ad979ee 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -283,7 +283,7 @@ def release_all_external_resources(self): """ self.discard_scorm_package(remove_scorm_pkg_root=True) - def discard_scorm_package(self, remove_scorm_pkg_root=False): + def discard_scorm_package(self, remove_scorm_pkg_root=False, expired_pkg_uuid=None): """Remove old scorm package 1) When admin uploading a new scorm package, we specify this argument `remove_scorm_pkg_root` with value `False`. @@ -294,34 +294,34 @@ def discard_scorm_package(self, remove_scorm_pkg_root=False): """ try: + _pkg_uuid = expired_pkg_uuid if expired_pkg_uuid else self.scorm_pkg.split('/')[0] + if not _pkg_uuid: + logger.info('[WARN] uuid {} not found in path.'.format(_pkg_uuid)) + return + if isinstance(self.fs, OSFS): - _pkg_uuid = self.scorm_pkg.split('/')[0] - if _pkg_uuid: - # Remove : .../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/2b1fe852ed9c4f59b1be5b7636be9f32 - _pkg_folder_path = self.fs.getsyspath(_pkg_uuid) - if remove_scorm_pkg_root: - _pkg_folder_path = _pkg_folder_path[:_pkg_folder_path.find('/fs/')] - - if path_exists(_pkg_folder_path): - remove_folder(_pkg_folder_path) - logger.info('[INFO] Old SCORM Package Folder {} removed.'.format(_pkg_folder_path)) - - # Remove : .../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/2b1fe852ed9c4f59b1be5b7636be9f32.zip - _zip_scorm_package = _pkg_folder_path + '.zip' - if path_exists(_zip_scorm_package): - remove_file(_zip_scorm_package) - logger.info('[INFO] Old SCORM zip package {} removed.'.format(_zip_scorm_package)) - - else: - logger.warn('[WARN] Old SCORM Package Folder {} not found.'.format(_pkg_folder_path)) + # Remove : .../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/2b1fe852ed9c4f59b1be5b7636be9f32 + _pkg_folder_path = self.fs.getsyspath(_pkg_uuid) + if remove_scorm_pkg_root: + _pkg_folder_path = _pkg_folder_path[:_pkg_folder_path.find('/fs/')] + + if path_exists(_pkg_folder_path): + remove_folder(_pkg_folder_path) + logger.info('[INFO] Old SCORM Package Folder {} removed.'.format(_pkg_folder_path)) + + # Remove : .../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/2b1fe852ed9c4f59b1be5b7636be9f32.zip + _zip_scorm_package = _pkg_folder_path + '.zip' + if path_exists(_zip_scorm_package): + remove_file(_zip_scorm_package) + logger.info('[INFO] Old SCORM zip package {} removed.'.format(_zip_scorm_package)) + else: - logger.info('[WARN] uuid not found in path {}.'.format(self.scorm_pkg)) + logger.warn('[WARN] Old SCORM Package Folder {} not found.'.format(_pkg_folder_path)) else: deleted_count = 0 S3_BUCKET_NAME = settings.DJFS.get('bucket') _s3_prefix = self.fs.dir_path[1:] # Sample: /xblock/block--v1-_beta-.Content__demo-.2020Q1-.type_64_scormxblock-.block_64_ae63e8b39db84405a8763c9a5441f93c/fs/NONE.NONE - _pkg_uuid = self.scorm_pkg.split('/')[0] _s3_prefix = _s3_prefix if remove_scorm_pkg_root else (_s3_prefix + '/' + _pkg_uuid) _kwargs = {'Bucket': S3_BUCKET_NAME, 'Prefix': _s3_prefix} logger.info('[INFO] Removing AWS S3 Old SCORM Packages by BucketName={}, PREFIX={}...'.format(S3_BUCKET_NAME, _s3_prefix)) @@ -363,6 +363,7 @@ def studio_upload_files(self, request, suffix=''): self.scorm_pkg_version, scorm_index, scorm_launch = self._get_scorm_info(mf) pkg_id = uuid.uuid4().hex + expired_pkg_uuid = self.scorm_pkg.split('/')[0] pkg_id = self._upload_scorm_pkg(pkg, pkg_id) self.scorm_pkg = os.path.join(pkg_id, scorm_index) @@ -374,7 +375,8 @@ def studio_upload_files(self, request, suffix=''): self._upload_scorm_zip(pkg.file, pkg_id) self.scorm_pkg_filename = pkg.filename - self.discard_scorm_package() + # Remove expired package + self.discard_scorm_package(expired_pkg_uuid=expired_pkg_uuid) if cover_images: self.cover_images = [ From 77269f8b7e12752bbf5fb7f875ccd04563ce2b90 Mon Sep 17 00:00:00 2001 From: Barry Bao Date: Fri, 2 Feb 2024 12:05:57 +0800 Subject: [PATCH 13/13] Fixed --- scormxblock/scormxblock.py | 59 +++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/scormxblock/scormxblock.py b/scormxblock/scormxblock.py index ad979ee..bb8c24f 100644 --- a/scormxblock/scormxblock.py +++ b/scormxblock/scormxblock.py @@ -46,6 +46,10 @@ from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers except ImportError: pass + +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import ItemNotFoundError from .scorm_default import * from .fields import DateTime from .mixins import ScorableXBlockMixin @@ -279,29 +283,39 @@ class ScormXBlock(StudioEditableXBlockMixin, ScorableXBlockMixin, XBlock): has_author_view = True def release_all_external_resources(self): - """Called when course/chapter/subsection/unit got removed. + """Called when course got removed. """ - self.discard_scorm_package(remove_scorm_pkg_root=True) + # Delete Draft + self.discard_scorm_package(self, remove_scorm_pkg_root=True) + # Delete The Published + try: + self.discard_scorm_package( + modulestore().get_item(self.location, revision=ModuleStoreEnum.RevisionOption.published_only), + remove_scorm_pkg_root=True + ) + except ItemNotFoundError: + logger.info('[INFO] published scorm instance does not exist : location_key : {}'.format(self.location)) - def discard_scorm_package(self, remove_scorm_pkg_root=False, expired_pkg_uuid=None): - """Remove old scorm package + @staticmethod + def discard_scorm_package(xblock, remove_scorm_pkg_root=False, expired_pkg_uuid=None): + """Remove scorm packages from draft or published mongodb. - 1) When admin uploading a new scorm package, we specify this argument `remove_scorm_pkg_root` with value `False`. + 1) When admin uploading a new scorm package, we specify the Flag(`remove_scorm_pkg_root`) with `False`. Then the folder `.../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/xxxx_uuid_xxxxx` will be cleared. - 2) When admin remove scorm from `Unit`, we specify this argument `remove_scorm_pkg_root` with value `True`. - Then the folder `.../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65` will be removed. + 2) When admin Remove scorm from `Unit` + Publish this unit, we specify the Flag(`remove_scorm_pkg_root`) with `True`. + Then the parent folder `.../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65` will be removed ( `Published` scorm package included inside ). """ try: - _pkg_uuid = expired_pkg_uuid if expired_pkg_uuid else self.scorm_pkg.split('/')[0] + _pkg_uuid = expired_pkg_uuid if expired_pkg_uuid else xblock.scorm_pkg.split('/')[0] if not _pkg_uuid: logger.info('[WARN] uuid {} not found in path.'.format(_pkg_uuid)) return - if isinstance(self.fs, OSFS): + if isinstance(xblock.fs, OSFS): # Remove : .../block--v1-_edX-.SLV__0001-.2023--10--10-.type_64_scormxblock-.block_64_c8c2f1fe2c9c4fed926b80942aab5a65/fs/NONE.NONE/2b1fe852ed9c4f59b1be5b7636be9f32 - _pkg_folder_path = self.fs.getsyspath(_pkg_uuid) + _pkg_folder_path = xblock.fs.getsyspath(_pkg_uuid) if remove_scorm_pkg_root: _pkg_folder_path = _pkg_folder_path[:_pkg_folder_path.find('/fs/')] @@ -321,19 +335,19 @@ def discard_scorm_package(self, remove_scorm_pkg_root=False, expired_pkg_uuid=No else: deleted_count = 0 S3_BUCKET_NAME = settings.DJFS.get('bucket') - _s3_prefix = self.fs.dir_path[1:] # Sample: /xblock/block--v1-_beta-.Content__demo-.2020Q1-.type_64_scormxblock-.block_64_ae63e8b39db84405a8763c9a5441f93c/fs/NONE.NONE + _s3_prefix = xblock.fs.dir_path[1:] # Sample: /xblock/block--v1-_beta-.Content__demo-.2020Q1-.type_64_scormxblock-.block_64_ae63e8b39db84405a8763c9a5441f93c/fs/NONE.NONE _s3_prefix = _s3_prefix if remove_scorm_pkg_root else (_s3_prefix + '/' + _pkg_uuid) _kwargs = {'Bucket': S3_BUCKET_NAME, 'Prefix': _s3_prefix} logger.info('[INFO] Removing AWS S3 Old SCORM Packages by BucketName={}, PREFIX={}...'.format(S3_BUCKET_NAME, _s3_prefix)) while True: - resp = self.fs.client.list_objects_v2(**_kwargs) + resp = xblock.fs.client.list_objects_v2(**_kwargs) _delete_keys = { 'Objects': [ {'Key': k if isinstance(k, unicode) else k.decode('utf-8')} for k in [obj['Key'] for obj in resp.get('Contents', [])] ] } - s3_resp = self.fs.client.delete_objects(Bucket=S3_BUCKET_NAME, Delete=_delete_keys) + s3_resp = xblock.fs.client.delete_objects(Bucket=S3_BUCKET_NAME, Delete=_delete_keys) _errors = s3_resp.get('Errors', None) if _errors: raise Exception(_errors) @@ -362,9 +376,25 @@ def studio_upload_files(self, request, suffix=''): mf = zip_fs.read('imsmanifest.xml') self.scorm_pkg_version, scorm_index, scorm_launch = self._get_scorm_info(mf) + only_used_in_draft = False pkg_id = uuid.uuid4().hex expired_pkg_uuid = self.scorm_pkg.split('/')[0] + try: + published_scorm = modulestore().get_item( + self.location, revision=ModuleStoreEnum.RevisionOption.published_only + ) + published_pkg_uuid = published_scorm.scorm_pkg.split('/')[0] + + if published_pkg_uuid != expired_pkg_uuid: + only_used_in_draft = True + logger.info('[INFO] Expired scorm package (uuid={}) only used by draft. So we can remove it.'.format(expired_pkg_uuid)) + else: + logger.info('[INFO] Scorm package (uuid={}) is being used by the published. So keep it.'.format(expired_pkg_uuid)) + + except ItemNotFoundError as e: + logger.info('[INFO] Scorm(Published) instance not found : detail : {}'.format(str(e))) + pkg_id = self._upload_scorm_pkg(pkg, pkg_id) self.scorm_pkg = os.path.join(pkg_id, scorm_index) self.scorm_pkg_modified = timezone.now() @@ -376,7 +406,8 @@ def studio_upload_files(self, request, suffix=''): self.scorm_pkg_filename = pkg.filename # Remove expired package - self.discard_scorm_package(expired_pkg_uuid=expired_pkg_uuid) + if only_used_in_draft: + self.discard_scorm_package(self, expired_pkg_uuid=expired_pkg_uuid) if cover_images: self.cover_images = [