Version 0.7.4

* Add options for mypy
--implicit-optional --allow-redefinition --disable-error-code abstract

* Fix typings

* Install setuptools firstly

* Use typing_extensions.Final for python3.7

* Disable attr-defined checking at windows

Author: PeterDing

.github/workflows/main.yml

@@ -24,12 +24,18 @@ jobs:
         with:
           virtualenvs-create: true
       - name: Install dependencies
-        run: poetry install --no-root
+        run: |
+          poetry run pip3 install setuptools
+          poetry install --no-root
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
       - name: Format check with black
         run: poetry run black --check .
-      - name: Typecheck with mypy
-        run: poetry run mypy -p baidupcs_py --ignore-missing-imports --warn-unreachable
+      - name: Typecheck with mypy at Windows
+        run: poetry run mypy -p baidupcs_py --ignore-missing-imports --warn-unreachable --implicit-optional --allow-redefinition --disable-error-code abstract --disable-error-code attr-defined
+        if: matrix.os == 'windows-latest'
+      - name: Typecheck with mypy at Linux
+        run: poetry run mypy -p baidupcs_py --ignore-missing-imports --warn-unreachable --implicit-optional --allow-redefinition --disable-error-code abstract
+        if: matrix.os == 'ubuntu-latest'
       - name: Test with pytest
         run: |
           poetry run python build.py build_ext --inplace

CHANGELOG.md

@@ -1,6 +1,16 @@
 # Changelog
 
-## v0.7.3 - 2022-10-31
+## v0.7.4 - 2022-11-10
+
+### Changed
+
+- 在下载和上传时，让调用者去初始化进度条。
+
+### Updated
+
+- 更新依赖。
+
+## v0.7.3 - 2022-11-09
 
 ### Updated
 

Makefile

@@ -1,5 +1,10 @@
 typecheck:
-	mypy -p baidupcs_py --ignore-missing-imports --warn-unreachable
+	mypy -p baidupcs_py \
+		--ignore-missing-imports \
+		--warn-unreachable \
+		--implicit-optional \
+		--allow-redefinition \
+		--disable-error-code abstract
 
 format-check:
 	black --check .
@@ -10,6 +15,8 @@ format:
 build-pyx:
 	python build.py build_ext --inplace
 
+test: build-pyx
+	pytest -s tests/test_common.py
 
 build: all
 	rm -fr dist

baidupcs_py/__init__.py

@@ -1,3 +1,3 @@
 from baidupcs_py.baidupcs import BaiduPCS, BaiduPCSApi
 
-__version__ = "0.7.3"
+__version__ = "0.7.4"

baidupcs_py/app/account.py

@@ -110,7 +110,7 @@ def cd(self, remotedir: str = "/"):
 
         assert account
 
-        pwd = join_path(account.pwd, remotedir)
+        pwd = join_path(Path(account.pwd), Path(remotedir))
         account = account._replace(pwd=pwd)
         self._accounts[self._who] = account
 

baidupcs_py/commands/log.py

@@ -1,13 +1,17 @@
+from typing import cast
 import os
 from pathlib import Path
 
-from baidupcs_py.common.log import LogLevels, get_logger as _get_logger
+from baidupcs_py.common.log import TLogLevel, LogLevels, get_logger as _get_logger
 from baidupcs_py.commands.env import LOG_LEVEL, LOG_PATH
 
 
 def get_logger(name: str):
     _LOG_PATH = Path(os.getenv("LOG_PATH") or LOG_PATH).expanduser()
-    _LOG_LEVEL = os.getenv("LOG_LEVEL", LOG_LEVEL).upper()
+    _LOG_LEVEL: TLogLevel = cast(
+        TLogLevel,
+        os.getenv("LOG_LEVEL", LOG_LEVEL).upper(),
+    )
     assert _LOG_LEVEL in LogLevels
 
     return _get_logger(name, filename=_LOG_PATH, level=_LOG_LEVEL)

baidupcs_py/commands/rapid_upload.py

@@ -1,5 +1,6 @@
 from typing import Optional, List, Dict, Any
 
+from pathlib import Path
 from threading import Semaphore
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
@@ -176,7 +177,7 @@ def rapid_upload(
         content_crc32 = content_crc32 or 0
         filename = filename or _filename
 
-    remotepath = join_path(remotedir, filename)
+    remotepath = join_path(Path(remotedir), Path(filename))
 
     assert all(
         [slice_md5, content_md5, content_length]

baidupcs_py/commands/server.py

@@ -61,7 +61,7 @@ async def handle_request(
 
     remotepath = remotepath.strip("/")
 
-    _rp = join_path(_root_dir, remotepath)
+    _rp = join_path(Path(_root_dir), Path(remotepath))
 
     # Anti path traversal attack
     if not _rp.startswith(_root_dir):

baidupcs_py/commands/sync.py

@@ -66,12 +66,12 @@ def sync(
     fts: List[FromTo] = []
     check_list: List[Tuple[str, PcsFile]] = []
     all_localpaths = set()
-    for localpath in walk(localdir):
+    for localpath in walk(Path(localdir)):
         path = localpath[len(localdir) + 1 :]
         all_localpaths.add(path)
 
         if path not in all_pcs_files:
-            fts.append(FromTo(localpath, join_path(remotedir, path)))
+            fts.append(FromTo(localpath, join_path(Path(remotedir), Path(path))))
         else:
             check_list.append((localpath, all_pcs_files[path]))
 

baidupcs_py/commands/upload.py

@@ -75,15 +75,15 @@ def from_tos(localpaths: List[str], remotedir: str) -> List[FromTo]:
 
     ft: List[FromTo] = []
     for localpath in localpaths:
-        if not exists(localpath):
+        if not exists(Path(localpath)):
             continue
 
-        if is_file(localpath):
+        if is_file(Path(localpath)):
             remotepath = to_remotepath(os.path.basename(localpath), remotedir)
             ft.append(FromTo(localpath, remotepath))
         else:
             parents_num = max(len(Path(localpath).parts) - 1, 0)
-            for sub_path in walk(localpath):
+            for sub_path in walk(Path(localpath)):
                 relative_path = Path(*Path(sub_path).parts[parents_num:]).as_posix()
                 remotepath = to_remotepath(relative_path, remotedir)
                 ft.append(FromTo(sub_path, remotepath))

baidupcs_py/common/downloader.py

@@ -1,4 +1,4 @@
-from typing import Optional, Any, Callable
+from typing import Optional, List, Any, Callable
 from os import PathLike
 from pathlib import Path
 from threading import Semaphore
@@ -14,6 +14,10 @@
 
 
 class MeDownloader(RangeRequestIO):
+    _executor: ThreadPoolExecutor
+    _semaphore: Semaphore
+    _futures: List[Future]
+
     @classmethod
     def _set_executor(
         cls,

baidupcs_py/common/io.py

@@ -1,4 +1,16 @@
-from typing import Optional, List, Tuple, Dict, Union, Any, Callable, Generator, IO
+from typing import (
+    Optional,
+    List,
+    Tuple,
+    Dict,
+    Union,
+    Any,
+    Callable,
+    Generator,
+    IO,
+    cast,
+    AnyStr,
+)
 from io import BytesIO, UnsupportedOperation
 from enum import Enum
 from pathlib import Path
@@ -29,15 +41,15 @@
     calu_md5,
     calu_crc32_and_md5,
 )
-from baidupcs_py.common.log import LogLevels, get_logger
+from baidupcs_py.common.log import TLogLevel, LogLevels, get_logger
 
-import requests
+import requests  # type: ignore
 from requests import Response
 
 _LOG_LEVEL = os.getenv("LOG_LEVEL", "CRITICAL").upper()
 if _LOG_LEVEL not in LogLevels:
     _LOG_LEVEL = "CRITICAL"
-logger = get_logger(__name__, level=_LOG_LEVEL)
+logger = get_logger(__name__, level=cast(TLogLevel, _LOG_LEVEL))
 
 READ_SIZE = 65535
 
@@ -239,11 +251,17 @@ def __init__(self, io: IO, encrypt_password: bytes, total_origin_len: int):
         # Cryptography
         #
         # Instantiated at each subclass, here is for mypy
-        self._crypto: Optional[Cryptography] = None
+        self._crypto: Cryptography
 
-        self._total_head = None
+        self._total_head: bytes
         self._total_head_len = len(self.total_head)
 
+    def magic_code(self) -> bytes:
+        raise NotImplementedError()
+
+    def block_size(self) -> int:
+        raise NotImplementedError()
+
     def reset(self):
         """Reset io and crypto"""
         self._io.seek(0, 0)
@@ -278,14 +296,14 @@ def total_head(self) -> bytes:
         `salt 8bytes`
         """
 
-        if self._total_head:
+        if hasattr(self, "_total_head"):
             return self._total_head
 
         assert len(self._salt) == 8
 
         ori_head = padding_key(
             BAIDUPCS_PY_CRYPTO_MAGIC_CODE
-            + self.MAGIC_CODE
+            + self.magic_code()
             + self._salt
             + random_sys_bytes(8)
             + u64_to_u8x8(self._total_origin_len),
@@ -375,6 +393,7 @@ def close(self):
 
 
 class SimpleEncryptIO(EncryptIO):
+
     MAGIC_CODE = b"\x00"
 
     BLOCK_SIZE = 16  # 16 bytes
@@ -383,6 +402,12 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._crypto = SimpleCryptography(self._encrypt_key + self._nonce_or_iv)
 
+    def magic_code(self) -> bytes:
+        return self.MAGIC_CODE
+
+    def block_size(self) -> int:
+        return self.BLOCK_SIZE
+
 
 class ChaCha20EncryptIO(EncryptIO):
     MAGIC_CODE = b"\x01"
@@ -399,6 +424,12 @@ def reset(self):
     def seekable(self) -> bool:
         return False
 
+    def magic_code(self) -> bytes:
+        return self.MAGIC_CODE
+
+    def block_size(self) -> int:
+        return self.BLOCK_SIZE
+
 
 class AES256CBCEncryptIO(EncryptIO):
     MAGIC_CODE = b"\x02"
@@ -422,6 +453,12 @@ def __init__(self, *args, **kwargs):
 
         self._need_data_padded = self._total_origin_len != self._encrypted_io_len
 
+    def magic_code(self) -> bytes:
+        return self.MAGIC_CODE
+
+    def block_size(self) -> int:
+        return self.BLOCK_SIZE
+
     def reset(self):
         super().reset()
         self._origin_io_offset = 0
@@ -677,7 +714,7 @@ def _read_block(self, size: int = -1):
             len(self._encrypted_cache), self.BLOCK_SIZE, ceil=False
         )
         if avail_enc_len > 0:
-            enc_cn = self._encrypted_cache[:avail_enc_len]
+            enc_cn = bytes(self._encrypted_cache[:avail_enc_len])
             self._encrypted_cache = self._encrypted_cache[avail_enc_len:]
         else:
             enc_cn = b""
@@ -731,15 +768,15 @@ def _decryptio_version1(
     encrypt_password = padding_key(encrypt_password, 32)
 
     if len(total_head) < ENCRYPT_HEAD_LEN:
-        return
+        return None
 
     # Version 1
-    b_mc, magic_code, nonce_or_iv, total_origin_len = parse_head(total_head)
+    b_mc, magic_code, nonce_or_iv, total_origin_len_ = parse_head(total_head)
     b_mc = aes256cbc_decrypt(b_mc, encrypt_password, random_bytes(16, encrypt_password))
-    total_origin_len = u8x8_to_u64(total_origin_len)
+    total_origin_len = u8x8_to_u64(total_origin_len_)
 
     if b_mc != BAIDUPCS_PY_CRYPTO_MAGIC_CODE:
-        return
+        return None
 
     if magic_code == SimpleEncryptIO.MAGIC_CODE:
         return SimpleDecryptIO(io, encrypt_password, b"", total_origin_len)
@@ -748,15 +785,15 @@ def _decryptio_version1(
     elif magic_code == AES256CBCEncryptIO.MAGIC_CODE:
         return AES256CBCDecryptIO(io, encrypt_password, nonce_or_iv, total_origin_len)
     else:
-        logging.warning(f"Unknown magic_code: {magic_code}")
-        return
+        logging.warning(f"Unknown magic_code: {magic_code!r}")
+        return None
 
 
 def _decryptio_version3(
     total_head: bytes, io: IO, encrypt_password: bytes
 ) -> Optional[DecryptIO]:
     if len(total_head) < PADDED_ENCRYPT_HEAD_WITH_SALT_LEN:
-        return
+        return None
 
     enc_head, salt_for_head = (
         total_head[:PADDED_ENCRYPT_HEAD_LEN],
@@ -769,25 +806,25 @@ def _decryptio_version3(
 
     head = aes256cbc_decrypt(enc_head, encrypt_key_for_head, iv_for_head)
 
-    b_mc, magic_code, padding_salt, total_origin_len = parse_head(head)
-    total_origin_len = u8x8_to_u64(total_origin_len)
+    b_mc, magic_code, padding_salt, total_origin_len_ = parse_head(head)
+    total_origin_len = u8x8_to_u64(total_origin_len_)
 
     salt = padding_salt[:8]
     encrypt_key, nonce_or_iv = generate_key_iv(encrypt_password, salt, 32, 16)
 
     if b_mc != BAIDUPCS_PY_CRYPTO_MAGIC_CODE:
-        return
+        return None
 
-    eio = None
+    eio: DecryptIO
     if magic_code == SimpleEncryptIO.MAGIC_CODE:
         eio = SimpleDecryptIO(io, encrypt_key, nonce_or_iv, total_origin_len)
     elif magic_code == ChaCha20EncryptIO.MAGIC_CODE:
         eio = ChaCha20DecryptIO(io, encrypt_key, nonce_or_iv, total_origin_len)
     elif magic_code == AES256CBCEncryptIO.MAGIC_CODE:
         eio = AES256CBCDecryptIO(io, encrypt_key, nonce_or_iv, total_origin_len)
     else:
-        logging.warning(f"Unknown magic_code: {magic_code}")
-        return
+        logging.warning(f"Unknown magic_code: {magic_code!r}")
+        return None
 
     eio._total_head_len = PADDED_ENCRYPT_HEAD_WITH_SALT_LEN
     return eio
@@ -832,7 +869,7 @@ def __init__(
         self._kwargs = kwargs
         self._max_chunk_size = max_chunk_size
         self._encrypt_password = encrypt_password
-        self._dio = None
+        self._dio: Optional[IO] = None
         self._has_encrypted = False
         self._total_head_len = 0
         self._decrypted_count = 0
@@ -906,7 +943,9 @@ def _parse_crypto(self):
                     self._dio = to_decryptio(BytesIO(raw_data), self._encrypt_password)
                     self._has_encrypted = isinstance(self._dio, DecryptIO)
                     self._total_head_len = (
-                        self._dio._total_head_len if self._has_encrypted else 0
+                        cast(DecryptIO, self._dio)._total_head_len
+                        if self._has_encrypted
+                        else 0
                     )
         self._parsed = True
 
@@ -966,6 +1005,7 @@ def __len__(self) -> int:
 
         self._init()
 
+        assert self._content_length
         return self._content_length - self._total_head_len
 
     def read(self, _range: Tuple[int, int]) -> Generator[bytes, None, None]: