Skip to content

增加上传开关秒传的能力 #138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,7 @@ BaiduPCS-Py upload --max-workers 4 [OPTIONS] [LOCALPATHS]... REMOTEDIR
| --no-ignore-existing, --NI | 上传已经存在的文件 |
| --no-show-progress, --NP | 不显示上传进度 |
| --check-md5, --CM | 分段上传后检查 md5。注意检查上传后大文件的 md5,可能会花数分中(2G 的文件需要大约 5 分钟) |
| --enable-rapid-upload, --R | 启用秒传,默认启用(秒传API有变动,如果上传失败,请关闭秒传后在试) |

## 同步本地目录到远端

Expand Down
6 changes: 6 additions & 0 deletions baidupcs_py/app/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -1183,6 +1183,7 @@ def play(
is_flag=True,
help="分段上传后检查md5。注意检查上传后大文件的md5,可能会花数分中(2G 的文件需要大约5分钟)",
)
@click.option("--enable-rapid-upload", "--R", is_flag=True, help="启用秒传")
@click.pass_context
@handle_error
@multi_user_do
Expand All @@ -1197,6 +1198,7 @@ def upload(
no_ignore_existing,
no_show_progress,
check_md5,
enable_rapid_upload,
):
"""上传文件"""

Expand Down Expand Up @@ -1236,6 +1238,7 @@ def upload(
user_id=user_id,
user_name=user_name,
check_md5=check_md5,
enable_rapid_upload=enable_rapid_upload,
)


Expand All @@ -1258,6 +1261,7 @@ def upload(
is_flag=True,
help="分段上传后检查md5。注意检查上传后大文件的md5,可能会花数分中(2G 的文件需要大约5分钟)",
)
@click.option("--enable-rapid-upload", "--R", is_flag=True, help="启用秒传")
@click.pass_context
@handle_error
@multi_user_do
Expand All @@ -1270,6 +1274,7 @@ def sync(
max_workers,
no_show_progress,
check_md5,
enable_rapid_upload,
):
"""同步本地目录到远端"""

Expand Down Expand Up @@ -1303,6 +1308,7 @@ def sync(
user_id=user_id,
user_name=user_name,
check_md5=check_md5,
enable_rapid_upload=enable_rapid_upload,
)


Expand Down
2 changes: 2 additions & 0 deletions baidupcs_py/commands/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ def sync(
user_id: Optional[int] = None,
user_name: Optional[str] = None,
check_md5: bool = False,
enable_rapid_upload: bool = True,
):
localdir = Path(localdir).as_posix()
remotedir = Path(remotedir).as_posix()
Expand Down Expand Up @@ -119,6 +120,7 @@ def sync(
user_id=user_id,
user_name=user_name,
check_md5=check_md5,
enable_rapid_upload=enable_rapid_upload,
)

if to_deletes:
Expand Down
13 changes: 11 additions & 2 deletions baidupcs_py/commands/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ def upload(
user_id: Optional[int] = None,
user_name: Optional[str] = None,
check_md5: bool = False,
enable_rapid_upload: bool = True,
):
"""Upload from_tos

Expand Down Expand Up @@ -175,6 +176,7 @@ def upload(
user_id=user_id,
user_name=user_name,
check_md5=check_md5,
enable_rapid_upload=enable_rapid_upload,
)
elif upload_type == UploadType.Many:
upload_many(
Expand All @@ -190,6 +192,7 @@ def upload(
user_id=user_id,
user_name=user_name,
check_md5=check_md5,
enable_rapid_upload=enable_rapid_upload,
)


Expand Down Expand Up @@ -335,6 +338,7 @@ def upload_one_by_one(
user_id: Optional[int] = None,
user_name: Optional[str] = None,
check_md5: bool = False,
enable_rapid_upload: bool = True,
):
"""Upload files one by one with uploading the slices concurrently"""

Expand All @@ -355,6 +359,7 @@ def upload_one_by_one(
user_id=user_id,
user_name=user_name,
check_md5=check_md5,
enable_rapid_upload=enable_rapid_upload,
)

logger.debug("======== Uploading end ========")
Expand Down Expand Up @@ -385,6 +390,7 @@ def upload_file_concurrently(
user_id: Optional[int] = None,
user_name: Optional[str] = None,
check_md5: bool = False,
enable_rapid_upload: bool = True,
):
"""Uploading one file by uploading it's slices concurrently"""

Expand Down Expand Up @@ -421,7 +427,7 @@ def callback_for_slice(idx: int, monitor: MultipartEncoderMonitor):
content_md5 = ""
content_crc32 = 0

if encrypt_type == EncryptType.No and encrypt_io_len > 256 * constant.OneK:
if encrypt_type == EncryptType.No and encrypt_io_len > 256 * constant.OneK and enable_rapid_upload:
# Rapid Upload
slice256k_md5, content_md5, content_crc32, _ = rapid_upload_params(encrypt_io)
ok = _rapid_upload(
Expand Down Expand Up @@ -560,6 +566,7 @@ def upload_many(
user_id: Optional[int] = None,
user_name: Optional[str] = None,
check_md5: bool = False,
enable_rapid_upload: bool = True,
):
"""Upload files concurrently that one file is with one connection"""

Expand Down Expand Up @@ -590,6 +597,7 @@ def upload_many(
user_id=user_id,
user_name=user_name,
check_md5=check_md5,
enable_rapid_upload=enable_rapid_upload,
)
futs[fut] = from_to

Expand Down Expand Up @@ -638,6 +646,7 @@ def upload_file(
user_id: Optional[int] = None,
user_name: Optional[str] = None,
check_md5: bool = False,
enable_rapid_upload: bool = True,
):
"""Upload one file with one connection"""

Expand Down Expand Up @@ -673,7 +682,7 @@ def callback_for_slice(monitor: MultipartEncoderMonitor):
content_md5 = ""
content_crc32 = 0

if encrypt_type == EncryptType.No and encrypt_io_len > 256 * constant.OneK:
if encrypt_type == EncryptType.No and encrypt_io_len > 256 * constant.OneK and enable_rapid_upload:
# Rapid Upload
slice256k_md5, content_md5, content_crc32, _ = rapid_upload_params(encrypt_io)
ok = _rapid_upload(
Expand Down