Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feat: Fix free disk space needs computation (Issue #96) #99

Merged
merged 5 commits into from
Apr 16, 2024
Merged

Feat: Fix free disk space needs computation (Issue #96) #99

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f00b557
Fix double counting of folder sizes when article IDs are explicitly p…
zoidy Apr 15, 2024
4dee44d
Include only curation folders in the total estimated bag size computa…
zoidy Apr 15, 2024
6d9223c
Fix crash when there are dangling symlinks, log error instead
zoidy Apr 16, 2024
f82cea8
Lint
zoidy Apr 16, 2024
c123af5
Lint
zoidy Apr 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions figshare/Article.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -504,14 +504,19 @@ def read_version_dirs_fun(self, read_version_dirs, version_dir, version_data):
"""
Get size of files of the given directory path
:param dir_path string path of dir where file size require to calculate.
:param include_only string include in the total only paths that contain this string. If ommitted, includes all paths.
:return size integer
"""
def get_file_size_of_given_path(self, dir_path):
def get_file_size_of_given_path(self, dir_path, include_only=""):
size = 0
for path, dirs, files in os.walk(dir_path):
for f in files:
fp = os.path.join(path, f)
size += os.path.getsize(fp)
if include_only in path:
for f in files:
fp = os.path.join(path, f)
try:
size += os.path.getsize(fp)
except Exception:
pass

return size

Expand Down Expand Up @@ -673,7 +678,7 @@ def __copy_files_ual_rdm(self, version_data, folder_name):
self.check_access_of_directories(curation_storage_location, "curation")

preservation_storage_location = self.preservation_storage_location
complete_folder_name = preservation_storage_location + folder_name + "/" + version_no + "/UAL_RDM"
complete_folder_name = os.path.join(preservation_storage_location, folder_name, version_no, "UAL_RDM")
dirs = os.listdir(curation_storage_location)
for dir in dirs:
if (dir not in self.exclude_dirs):
Expand All @@ -686,15 +691,15 @@ def __copy_files_ual_rdm(self, version_data, folder_name):
for dir in read_dirs:
if dir not in self.exclude_dirs:
if (dir == version_no):
curation_dir_name = article_dir_in_curation + "/" + dir + "/UAL_RDM"
curation_dir_name = os.path.join(article_dir_in_curation, dir, "UAL_RDM")
# check preservation dir is reachable
self.check_access_of_directories(preservation_storage_location, "preservation")
try:
check_path_exists = os.path.exists(complete_folder_name)
if (check_path_exists is False):
os.makedirs(complete_folder_name, exist_ok=True)
# copying files to preservation version folder
shutil.copytree(curation_dir_name, complete_folder_name, dirs_exist_ok=True)
shutil.copytree(curation_dir_name, complete_folder_name, dirs_exist_ok=True, ignore_dangling_symlinks=False)
self.logs.write_log_in_file("info", "Copied curation files to preservation folder.", True)
result = True
except Exception as e:
Expand Down Expand Up @@ -729,7 +734,7 @@ def find_matched_articles(self, articles):
self.no_matched += 1
self.article_match_info[i] = f"article {data['id']} {version_no} ----- {data['author_dir']}"
if (self.input_articles_id):
self.matched_curation_folder_list.append(data['author_dir'])
self.matched_curation_folder_list.append(os.path.join(data['author_dir'], version_no))
else:
self.article_non_match_info[i] = f"article {data['id']} {version_no}"

Expand Down Expand Up @@ -904,9 +909,9 @@ def process_articles(self, articles):
curation_folder_size = 0
for folder in self.matched_curation_folder_list:
path = curation_storage_location + folder
curation_folder_size += self.get_file_size_of_given_path(path)
curation_folder_size += self.get_file_size_of_given_path(path, "UAL_RDM")
else:
curation_folder_size = self.get_file_size_of_given_path(curation_storage_location)
curation_folder_size = self.get_file_size_of_given_path(curation_storage_location, "UAL_RDM")

required_space = curation_folder_size + self.total_all_articles_file_size

Expand Down
Loading