From 05411c0b43f55dab33998e507e9cfb08b08168a7 Mon Sep 17 00:00:00 2001 From: Gijs Molenaar Date: Wed, 13 Apr 2022 13:53:51 +0200 Subject: [PATCH] Add some more checks and fix them, also don't leak password (fixes #104 and #93) (#105) * pass default pylama checks * fix formatting error * just use flake8 * fix spacing * fix more warnings * fix issue #93 * make mypy happy * fix doc build by slight restructure * prepare for 1.6.0 * doc shuffle * Document handle_download * remove one more empty line Co-authored-by: Joeri van Ruth --- CHANGES | 10 +++ Makefile | 17 +++- doc/api.rst | 8 +- pymonetdb/__init__.py | 15 ++-- pymonetdb/filetransfer/__init__.py | 99 ++++------------------ pymonetdb/filetransfer/directoryhandler.py | 9 +- pymonetdb/filetransfer/downloads.py | 15 ++++ pymonetdb/filetransfer/uploads.py | 58 +++++++++++-- pymonetdb/mapi.py | 56 ++++++------ pymonetdb/profiler.py | 2 +- pymonetdb/sql/cursors.py | 5 +- pymonetdb/sql/debug.py | 3 +- setup.py | 3 +- tests/install_monetdb_from_msi_dir.py | 1 - tests/test_capabilities.py | 2 +- tests/test_dbapi2.py | 1 - tests/test_filetransfer.py | 5 +- tests/test_mapi_uri.py | 17 ++-- tests/test_oid.py | 2 +- tests/test_udf.py | 2 +- tests/util.py | 1 - tests/windows_tests.py | 2 +- 22 files changed, 170 insertions(+), 163 deletions(-) diff --git a/CHANGES b/CHANGES index d8eb20af..a4b226a3 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,13 @@ +# 1.6.0 + +changes since 1.5.1 + +* Support for COPY INTO ON CLIENT (#57) +* Connection object leaks the password (#93) +* Make code style checking stricter (mypy and flake8) (#104) +* Make python UDF debug code work with Python3 (#45 ) + + # 1.5.1 changes since 1.5.0 diff --git a/Makefile b/Makefile index 463189fc..86a0610c 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,11 @@ DBFARM=dbfarm DATABASE=demo -.PHONY: doc clean test wheel sdist dbfarm-start database-init +.PHONY: doc clean test wheel sdist dbfarm-start database-init all build -all: test +all: test doc checks build + +build: wheel sdist venv/: python3 -m venv venv @@ -51,7 +53,7 @@ upload: venv/bin/twine wheel sdist venv/bin/twine upload dist/*.whl dist/*.tar.gz doc: setup - PATH=$${PATH}:${CURDIR}/venv/bin $(MAKE) -C doc html + PATH=$${PATH}:${CURDIR}/venv/bin $(MAKE) -C doc html SPHINXOPTS="-W" venv/bin/flake8: setup venv/bin/pip install flake8 @@ -59,7 +61,14 @@ venv/bin/flake8: setup flake8: venv/bin/flake8 venv/bin/flake8 --count --select=E9,F63,F7,F82 --show-source --statistics pymonetdb tests - venv/bin/flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics pymonetdb tests + venv/bin/flake8 --count --max-complexity=10 --max-line-length=127 --statistics pymonetdb tests + +venv/bin/pylama: setup + venv/bin/pip install "pylama[all]" + touch venv/bin/pylama + +pylama: venv/bin/pylama + venv/bin/pylama pymonetdb tests checks: mypy pycodestyle flake8 diff --git a/doc/api.rst b/doc/api.rst index f1897146..bfcd62c9 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -44,15 +44,9 @@ File Uploads and Downloads Classes related to file transfer requests as used by COPY INTO ON CLIENT. .. automodule:: pymonetdb.filetransfer - :members: Uploader, Downloader, SafeDirectoryHandler + :members: Upload, Uploader, Download, Downloader, SafeDirectoryHandler :member-order: bysource -.. automodule:: pymonetdb.filetransfer.uploads - :members: Upload - -.. automodule:: pymonetdb.filetransfer.downloads - :members: Download - MonetDB remote control ====================== diff --git a/pymonetdb/__init__.py b/pymonetdb/__init__.py index c039bdab..b39d0e70 100644 --- a/pymonetdb/__init__.py +++ b/pymonetdb/__init__.py @@ -14,15 +14,15 @@ from pymonetdb import sql from pymonetdb import mapi from pymonetdb import exceptions -from pymonetdb import profiler from pymonetdb.profiler import ProfilerConnection from pymonetdb.sql.connections import Connection -from pymonetdb.sql.pythonize import * -from pymonetdb.exceptions import * -from pymonetdb.filetransfer import Downloader, Uploader -from pymonetdb.filetransfer.downloads import Download -from pymonetdb.filetransfer.uploads import Upload +from pymonetdb.sql.pythonize import BINARY, Binary, DATE, Date, Time, Timestamp, DateFromTicks, TimestampFromTicks, \ + TimeFromTicks, NUMBER, ROWID, STRING, TIME, types, DATETIME, TimeTzFromTicks, TimestampTzFromTicks +from pymonetdb.exceptions import Error, DataError, DatabaseError, IntegrityError, InterfaceError, InternalError, \ + NotSupportedError, OperationalError, ProgrammingError, Warning +from pymonetdb.filetransfer.downloads import Download, Downloader +from pymonetdb.filetransfer.uploads import Upload, Uploader from pymonetdb.filetransfer.directoryhandler import SafeDirectoryHandler try: @@ -38,7 +38,8 @@ 'Timestamp', 'DateFromTicks', 'TimeFromTicks', 'TimestampFromTicks', 'DataError', 'DatabaseError', 'Error', 'IntegrityError', 'InterfaceError', 'InternalError', 'NUMBER', 'NotSupportedError', 'OperationalError', 'ProgrammingError', 'ROWID', 'STRING', 'TIME', 'Warning', 'apilevel', 'connect', 'paramstyle', - 'threadsafety', 'Download', 'Downloader', 'Upload', 'Uploader', 'SafeDirectoryHandler'] + 'threadsafety', 'Download', 'Downloader', 'Upload', 'Uploader', 'SafeDirectoryHandler', 'types', 'DATETIME', + 'TimeTzFromTicks', 'TimestampTzFromTicks'] def connect(*args, **kwargs) -> Connection: diff --git a/pymonetdb/filetransfer/__init__.py b/pymonetdb/filetransfer/__init__.py index 0b4cc37c..fc4b8430 100644 --- a/pymonetdb/filetransfer/__init__.py +++ b/pymonetdb/filetransfer/__init__.py @@ -3,16 +3,21 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # # Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V. +import typing - -from abc import ABC, abstractmethod -from pymonetdb import mapi as mapi_protocol from pymonetdb.exceptions import ProgrammingError -import pymonetdb.filetransfer.uploads -import pymonetdb.filetransfer.downloads +from .uploads import Uploader, Upload +from .downloads import Downloader, Download +from .directoryhandler import SafeDirectoryHandler + +if typing.TYPE_CHECKING: + from pymonetdb.mapi import Connection + +# these are used in the code but they are referred to in the docs +(Uploader, Downloader, SafeDirectoryHandler) -def handle_file_transfer(mapi: "mapi_protocol.Connection", cmd: str): +def handle_file_transfer(mapi: "Connection", cmd: str): if cmd.startswith("r "): parts = cmd[2:].split(' ', 2) if len(parts) == 2: @@ -33,12 +38,12 @@ def handle_file_transfer(mapi: "mapi_protocol.Connection", cmd: str): mapi._putblock(f"Invalid file transfer command: {cmd!r}") -def handle_upload(mapi: "mapi_protocol.Connection", filename: str, text_mode: bool, offset: int): +def handle_upload(mapi: "Connection", filename: str, text_mode: bool, offset: int): if not mapi.uploader: mapi._putblock("No upload handler has been registered with pymonetdb\n") return skip_amount = offset - 1 if offset > 0 else 0 - upload = pymonetdb.filetransfer.uploads.Upload(mapi) + upload = Upload(mapi) try: mapi.uploader.handle_upload(upload, filename, text_mode, skip_amount) except Exception as e: @@ -54,11 +59,11 @@ def handle_upload(mapi: "mapi_protocol.Connection", filename: str, text_mode: bo raise ProgrammingError("Upload handler didn't do anything") -def handle_download(mapi: "mapi_protocol.Connection", filename: str, text_mode: bool): +def handle_download(mapi: "Connection", filename: str, text_mode: bool): if not mapi.downloader: mapi._putblock("No download handler has been registered with pymonetdb\n") return - download = pymonetdb.filetransfer.downloads.Download(mapi) + download = Download(mapi) try: mapi.downloader.handle_download(download, filename, text_mode) except Exception as e: @@ -83,77 +88,3 @@ def handle_download(mapi: "mapi_protocol.Connection", filename: str, text_mode: raise e finally: download.close() - - -class Uploader(ABC): - """ - Base class for upload hooks. Instances of subclasses of this class can be - registered using pymonetdb.Connection.set_uploader(). Every time an upload - request is received, an Upload object is created and passed to this objects - .handle_upload() method. - - If the server cancels the upload halfway, the .cancel() methods is called - and all further data written is ignored. - """ - - @abstractmethod - def handle_upload(self, upload: "pymonetdb.filetransfer.uploads.Upload", filename: str, text_mode: bool, skip_amount: int): - """ - Called when an upload request is received. Implementations should either - send an error using upload.send_error(), or request a writer using - upload.text_writer() or upload.binary_writer(). All data written to the - writer will be sent to the server. - - Parameter 'filename' is the file name used in the COPY INTO statement. - Parameter 'text_mode' indicates whether the server requested a text file - or a binary file. In case of a text file, 'skip_amount' indicates the - number of lines to skip. In binary mode, 'skip_amount' is always 0. - - SECURITY NOTE! Make sure to carefully validate the file name before - opening files on the file system. Otherwise, if an adversary has taken - control of the network connection or of the server, they can use file - upload requests to read arbitrary files from your computer - (../../) - - """ - pass - - def cancel(self): - """Optional method called when the server cancels the upload.""" - pass - - -class Downloader(ABC): - """ - Base class for download hooks. Instances of subclasses of this class can be - registered using pymonetdb.Connection.set_downloader(). Every time a - download request arrives, a Download object is created and passed to this - objects .handle_download() method. - - SECURITY NOTE! Make sure to carefully validate the file name before opening - files on the file system. Otherwise, if an adversary has taken control of - the network connection or of the server, they can use download requests to - OVERWRITE ARBITRARY FILES on your computer - """ - - @abstractmethod - def handle_download(self, download: "pymonetdb.filetransfer.downloads.Download", filename: str, text_mode: bool): - """ - Called when a download request is received from the server. Implementations - should either refuse by sending an error using download.send_error(), or - request a reader using download.binary_reader() or download.text_reader(). - - Parameter 'filename' is the file name used in the COPY INTO statement. - Parameter 'text_mode' indicates whether the server requested to send a binary - file or a text file. - - SECURITY NOTE! Make sure to carefully validate the file name before opening - files on the file system. Otherwise, if an adversary has taken control of - the network connection or of the server, they can use download requests to - OVERWRITE ARBITRARY FILES on your computer - """ - pass - - -# Only import this at the end to avoid circular imports -from pymonetdb.filetransfer.directoryhandler import SafeDirectoryHandler \ No newline at end of file diff --git a/pymonetdb/filetransfer/directoryhandler.py b/pymonetdb/filetransfer/directoryhandler.py index 539d92d9..a722ccf4 100644 --- a/pymonetdb/filetransfer/directoryhandler.py +++ b/pymonetdb/filetransfer/directoryhandler.py @@ -13,9 +13,8 @@ from pathlib import Path from shutil import copyfileobj from typing import Optional -from pymonetdb.filetransfer import Uploader, Downloader -from pymonetdb.filetransfer.uploads import Upload -from pymonetdb.filetransfer.downloads import Download +from pymonetdb.filetransfer.uploads import Upload, Uploader +from pymonetdb.filetransfer.downloads import Download, Downloader class SafeDirectoryHandler(Uploader, Downloader): @@ -62,7 +61,7 @@ def secure_resolve(self, filename: str) -> Optional[Path]: else: return None - def handle_upload(self, upload: Upload, filename: str, text_mode: bool, skip_amount: int): + def handle_upload(self, upload: Upload, filename: str, text_mode: bool, skip_amount: int): # noqa: C901 """:meta private:""" # keep the API docs cleaner, this has already been documented on class Uploader. p = self.secure_resolve(filename) @@ -162,4 +161,4 @@ def lookup_compression_algorithm(filename: str): else: return open opener = import_module(mod).open - return opener \ No newline at end of file + return opener diff --git a/pymonetdb/filetransfer/downloads.py b/pymonetdb/filetransfer/downloads.py index 1b7e6deb..02e4cc97 100644 --- a/pymonetdb/filetransfer/downloads.py +++ b/pymonetdb/filetransfer/downloads.py @@ -129,4 +129,19 @@ class Downloader(ABC): @abstractmethod def handle_download(self, download: Download, filename: str, text_mode: bool): + """ + Called when a download request is received. Implementations should either + send an error using download.send_error(), or request a reader using + download.text_reader() or download.binary_reader(). + + Parameter 'filename' is the file name used in the COPY INTO statement. + Parameter 'text_mode' indicates whether the server requested text + or binary mode. + + SECURITY NOTE! Make sure to carefully validate the file name before + opening files on the file system. Otherwise, if an adversary has taken + control of the network connection or of the server, they can use file + download requests to overwrite arbitrary files on your computer. + (../../) + """ pass diff --git a/pymonetdb/filetransfer/uploads.py b/pymonetdb/filetransfer/uploads.py index 9b581f14..2311828b 100644 --- a/pymonetdb/filetransfer/uploads.py +++ b/pymonetdb/filetransfer/uploads.py @@ -6,20 +6,23 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # # Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V. - - +import typing from io import BufferedIOBase, BufferedWriter, RawIOBase, TextIOBase, TextIOWrapper +from abc import ABC, abstractmethod from typing import Any, Optional, Union -from pymonetdb import mapi as mapi_protocol +from pymonetdb.mapi import MSG_MORE, MSG_FILETRANS from pymonetdb.exceptions import ProgrammingError +if typing.TYPE_CHECKING: + from pymonetdb.mapi import Connection as MapiConnection + class Upload: """ Represents a request from the server to upload data to the server. It is passed to the Uploader registered by the application, which for example might retrieve the data from a file on the client system. See - pymonetdb.Connection.set_uploader(). + pymonetdb.sql.connections.Connection.set_uploader(). Use the method send_error() to refuse the upload, binary_writer() to get a binary file object to write to, or text_writer() to get a text-mode file @@ -29,7 +32,7 @@ class Upload: opening any files on the client system! """ - mapi: Optional["mapi_protocol.Connection"] + mapi: Optional["MapiConnection"] error = False cancelled = False bytes_sent = 0 @@ -39,7 +42,7 @@ class Upload: writer: Optional[BufferedWriter] = None twriter: Optional[TextIOBase] = None - def __init__(self, mapi: "mapi_protocol.Connection"): + def __init__(self, mapi: "MapiConnection"): self.mapi = mapi def _check_usable(self): @@ -141,10 +144,10 @@ def _send_and_get_prompt(self, data: Union[bytes, memoryview]) -> bool: assert self.mapi self._send(data, True) prompt = self.mapi._getblock() - if prompt == mapi_protocol.MSG_MORE: + if prompt == MSG_MORE: self.chunk_used = 0 return True - elif prompt == mapi_protocol.MSG_FILETRANS: + elif prompt == MSG_FILETRANS: # server says stop return False else: @@ -170,7 +173,7 @@ def close(self): self.mapi._putblock('') # receive acknowledgement resp = self.mapi._getblock() - if resp != mapi_protocol.MSG_FILETRANS: + if resp != MSG_FILETRANS: raise ProgrammingError(f"Unexpected server response: {resp[:50]!r}") self.mapi = None @@ -243,3 +246,40 @@ def close(self): self.pending = False return self.inner.close() + +class Uploader(ABC): + """ + Base class for upload hooks. Instances of subclasses of this class can be + registered using pymonetdb.Connection.set_uploader(). Every time an upload + request is received, an Upload object is created and passed to this objects + .handle_upload() method. + + If the server cancels the upload halfway, the .cancel() methods is called + and all further data written is ignored. + """ + + @abstractmethod + def handle_upload(self, upload: Upload, filename: str, text_mode: bool, + skip_amount: int): + """ + Called when an upload request is received. Implementations should either + send an error using upload.send_error(), or request a writer using + upload.text_writer() or upload.binary_writer(). All data written to the + writer will be sent to the server. + + Parameter 'filename' is the file name used in the COPY INTO statement. + Parameter 'text_mode' indicates whether the server requested a text file + or a binary file. In case of a text file, 'skip_amount' indicates the + number of lines to skip. In binary mode, 'skip_amount' is always 0. + + SECURITY NOTE! Make sure to carefully validate the file name before + opening files on the file system. Otherwise, if an adversary has taken + control of the network connection or of the server, they can use file + upload requests to read arbitrary files from your computer + (../../) + """ + pass + + def cancel(self): + """Optional method called when the server cancels the upload.""" + pass diff --git a/pymonetdb/mapi.py b/pymonetdb/mapi.py index 6ab5a04d..7c35f0ba 100644 --- a/pymonetdb/mapi.py +++ b/pymonetdb/mapi.py @@ -13,12 +13,16 @@ import struct import hashlib import os +import typing from typing import Optional, Tuple from urllib.parse import urlparse, parse_qs from pymonetdb.exceptions import OperationalError, DatabaseError, \ ProgrammingError, NotSupportedError, IntegrityError -import pymonetdb.filetransfer + +if typing.TYPE_CHECKING: + from pymonetdb.filetransfer.downloads import Downloader + from pymonetdb.filetransfer.uploads import Uploader logger = logging.getLogger(__name__) @@ -92,7 +96,6 @@ def __init__(self): self.hostname = "" self.port = 0 self.username = "" - self.password = "" self.database = "" self.language = "" self.handshake_options = None @@ -101,8 +104,9 @@ def __init__(self): self.downloader = None self.stashed_buffer = None - def connect(self, database, username, password, language, hostname=None, - port=None, unix_socket=None, connect_timeout=-1, handshake_options=None): + def connect(self, database: str, username: str, password: str, language: str, # noqa: C901 + hostname: Optional[str] = None, port: Optional[int] = None, unix_socket=None, connect_timeout=-1, + handshake_options=None): """ setup connection to MAPI server unix_socket is used if hostname is not defined. @@ -110,7 +114,8 @@ def connect(self, database, username, password, language, hostname=None, if ':' in database: if not database.startswith('mapi:monetdb:'): - raise DatabaseError("colon not allowed in database name, except as part of mapi:monetdb://[:]/ URI") + raise DatabaseError("colon not allowed in database name, except as part of " + "mapi:monetdb://[:]/ URI") parsed = urlparse(database[5:]) # parse basic settings if parsed.hostname or parsed.port: @@ -141,10 +146,10 @@ def connect(self, database, username, password, language, hostname=None, # Future work: parse other parameters such as reply_size. if hostname and hostname[:1] == '/' and not unix_socket: - unix_socket = '%s/.s.monetdb.%d' % (hostname, port) + unix_socket = f'{hostname}/.s.monetdb.{port}' hostname = None - if not unix_socket and os.path.exists("/tmp/.s.monetdb.%i" % port): - unix_socket = "/tmp/.s.monetdb.%i" % port + if not unix_socket and os.path.exists(f"/tmp/.s.monetdb.{port}"): + unix_socket = f"/tmp/.s.monetdb.{port}" elif not unix_socket and not hostname: hostname = 'localhost' @@ -156,7 +161,6 @@ def connect(self, database, username, password, language, hostname=None, self.hostname = hostname self.port = port self.username = username - self.password = password self.database = database self.language = language self.unix_socket = unix_socket @@ -197,17 +201,17 @@ def connect(self, database, username, password, language, hostname=None, if not (self.language == 'control' and not self.hostname): # control doesn't require authentication over socket - self._login() + self._login(password=password) self.socket.settimeout(socket.getdefaulttimeout()) self.state = STATE_READY - def _login(self, iteration=0): + def _login(self, password: str, iteration=0): """ Reads challenge from line, generate response and check if everything is okay """ challenge = self._getblock() - response = self._challenge_response(challenge) + response = self._challenge_response(challenge, password) self._putblock(response) prompt = self._getblock().strip() @@ -230,7 +234,7 @@ def _login(self, iteration=0): if redirect[1] == "merovingian": logger.debug("restarting authentication") if iteration <= 10: - self._login(iteration=iteration + 1) + self._login(iteration=iteration + 1, password=password) else: raise OperationalError("maximal number of redirects " "reached (10)") @@ -241,9 +245,10 @@ def _login(self, iteration=0): self.port = int(self.port) logger.info("redirect to monetdb://%s:%s/%s" % (self.hostname, self.port, self.database)) - self.socket.close() + if self.socket: + self.socket.close() self.connect(hostname=self.hostname, port=self.port, - username=self.username, password=self.password, + username=self.username, password=password, database=self.database, language=self.language) else: @@ -275,7 +280,7 @@ def _sabotage(self): # don't care pass - def cmd(self, operation): + def cmd(self, operation): # noqa: C901 """ put a mapi command on the line""" logger.debug("executing command %s" % operation) @@ -320,7 +325,7 @@ def cmd(self, operation): else: raise ProgrammingError("unknown state: %s" % response) - def _challenge_response(self, challenge): + def _challenge_response(self, challenge: str, password: str): # noqa: C901 """ generate a response to a mapi login challenge """ challenges = challenge.split(':') @@ -329,7 +334,6 @@ def _challenge_response(self, challenge): challenges.pop() salt, identity, protocol, hashes, endian = challenges[:5] - password = self.password if protocol == '9': algo = challenges[5] @@ -342,15 +346,15 @@ def _challenge_response(self, challenge): else: raise NotSupportedError("We only speak protocol v9") - for h in hashes.split(","): + for i in hashes.split(","): try: - s = hashlib.new(h) + s = hashlib.new(i) except ValueError: pass else: s.update(password.encode()) s.update(salt.encode()) - pwhash = "{" + h + "}" + s.hexdigest() + pwhash = "{" + i + "}" + s.hexdigest() break else: raise NotSupportedError("Unsupported hash algorithms required" @@ -380,6 +384,10 @@ def _getblock_and_transfer_files(self): """ read one mapi encoded block and take care of any file transfers the server requests""" buffer = self._get_buffer() offset = 0 + + # import this here to solve circular import + from pymonetdb.filetransfer import handle_file_transfer + while True: old_offset = offset offset = self._getblock_raw(buffer, old_offset) @@ -388,7 +396,7 @@ def _getblock_and_transfer_files(self): # File transfer request. Chop the cmd off the buffer by lowering the offset cmd = str(buffer[i + 1: offset - 1], 'utf-8') offset = i - 2 - pymonetdb.filetransfer.handle_file_transfer(self, cmd) + handle_file_transfer(self, cmd) continue else: break @@ -492,11 +500,11 @@ def set_reply_size(self, size): self.cmd("Xreply_size %s" % size) - def set_uploader(self, uploader: "pymonetdb.filetransfer.Uploader"): + def set_uploader(self, uploader: "Uploader"): """Register the given Uploader, or None to deregister""" self.uploader = uploader - def set_downloader(self, downloader: "pymonetdb.filetransfer.Downloader"): + def set_downloader(self, downloader: "Downloader"): """Register the given Downloader, or None to deregister""" self.downloader = downloader diff --git a/pymonetdb/profiler.py b/pymonetdb/profiler.py index 83b70665..9e622aa2 100644 --- a/pymonetdb/profiler.py +++ b/pymonetdb/profiler.py @@ -28,7 +28,7 @@ def connect(self, database, username="monetdb", password="monetdb", hostname=Non self._mapi.cmd("profiler.setheartbeat(%d);\n" % heartbeat) try: self._mapi.cmd("profiler.openstream();\n") - except OperationalError as e: + except OperationalError: # We might be talking to an older version of MonetDB. Try connecting # the old way. self._mapi.cmd("profiler.openstream(3);\n") diff --git a/pymonetdb/sql/cursors.py b/pymonetdb/sql/cursors.py index f53b1977..b98dbf7a 100644 --- a/pymonetdb/sql/cursors.py +++ b/pymonetdb/sql/cursors.py @@ -133,8 +133,7 @@ def close(self): pass self.connection = None - def execute(self, operation, parameters=None): - # type: (str, Optional[Dict]) -> int + def execute(self, operation: str, parameters: Optional[Dict] = None): """Prepare and execute a database operation (query or command). Parameters may be provided as mapping and will be bound to variables in the operation. @@ -344,7 +343,7 @@ def next(self): def __next__(self): return self.next() - def _store_result(self, block): + def _store_result(self, block): # noqa: C901 """ parses the mapi result into a resultset""" if not block: diff --git a/pymonetdb/sql/debug.py b/pymonetdb/sql/debug.py index 598f8728..0f963fe2 100644 --- a/pymonetdb/sql/debug.py +++ b/pymonetdb/sql/debug.py @@ -48,8 +48,7 @@ def execute(self, query): return result -def debug(cursor, query, fname, sample=-1): - # type: (Cursor, str, str, int) -> Any +def debug(cursor: "Cursor", query: str, fname: str, sample: int = -1) -> Any: """ Locally debug a given Python UDF function in a SQL query using the PDB debugger. Optionally can run on only a sample of the input data, for faster data export. diff --git a/setup.py b/setup.py index 4e1ad1ea..0cc30c6e 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() -__version__ = '1.5.1' +__version__ = '1.6.0' setup( name='pymonetdb', @@ -41,7 +41,6 @@ def read(fname): "Intended Audience :: Developers", "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/tests/install_monetdb_from_msi_dir.py b/tests/install_monetdb_from_msi_dir.py index 472dc697..ac8fb717 100644 --- a/tests/install_monetdb_from_msi_dir.py +++ b/tests/install_monetdb_from_msi_dir.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -from shutil import copyfile import shutil import sys diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index e5e68596..9c0690c5 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -116,7 +116,7 @@ def generator(row, col): self.assertTrue(len(rows) == 1, "ROLLBACK didn't work") self.cursor.execute('drop table %s' % self.table) - def test_truncation(self): + def test_truncation(self): # noqa: C901 columndefs = ('col1 INT', 'col2 VARCHAR(255)') def generator(row, col): diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index b8fe8742..f621fb7d 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -14,7 +14,6 @@ # import unittest import time -import sys import pymonetdb from tests.util import test_args diff --git a/tests/test_filetransfer.py b/tests/test_filetransfer.py index d182633e..631a0c78 100644 --- a/tests/test_filetransfer.py +++ b/tests/test_filetransfer.py @@ -45,7 +45,7 @@ class MyUploader(Uploader): # cancelled_at: Optional[int] = None - def handle_upload(self, upload: Upload, filename: str, text_mode: bool, skip_amount: int): + def handle_upload(self, upload: Upload, filename: str, text_mode: bool, skip_amount: int): # noqa: C901 if self.do_nothing_at_all: return elif filename.startswith("x"): @@ -217,7 +217,8 @@ def expect1(self, value): self.expect([(value,)]) def compression_prefix(self, scheme): - return {'gz': b'\x1F\x8B', 'bz2': b'\x42\x5A\x68', 'xz': b'\xFD\x37\x7A\x58\x5A\x00', 'lz4': b'\x04\x22\x4D\x18', None: None}[scheme] + return {'gz': b'\x1F\x8B', 'bz2': b'\x42\x5A\x68', 'xz': b'\xFD\x37\x7A\x58\x5A\x00', + 'lz4': b'\x04\x22\x4D\x18', None: None}[scheme] class TestFileTransfer(TestCase, Common): diff --git a/tests/test_mapi_uri.py b/tests/test_mapi_uri.py index ffd5a3dc..bc472f44 100644 --- a/tests/test_mapi_uri.py +++ b/tests/test_mapi_uri.py @@ -25,26 +25,30 @@ def attempt_connect(self, uri, username=None, password=None): cursor = connection.cursor() q = "select tag from sys.queue()" cursor.execute(q) - rows = cursor.fetchall() + cursor.fetchall() def test_no_uri(self): # test setUp and attempt_connect themselves self.attempt_connect(self.database, username=self.username, password=self.password) def test_full_mapi_uri(self): - self.attempt_connect(f"mapi:monetdb://{self.hostname}:{self.port}/{self.database}", username=self.username, password=self.password) + self.attempt_connect(f"mapi:monetdb://{self.hostname}:{self.port}/{self.database}", + username=self.username, password=self.password) def test_without_port(self): - self.attempt_connect(f"mapi:monetdb://{self.hostname}/{self.database}", username=self.username, password=self.password) + self.attempt_connect(f"mapi:monetdb://{self.hostname}/{self.database}", + username=self.username, password=self.password) def test_username_component(self): try: - self.attempt_connect(f"mapi:monetdb://{self.hostname}:{self.port}/{self.database}", username="not" + self.username, password="not" + self.password) + self.attempt_connect(f"mapi:monetdb://{self.hostname}:{self.port}/{self.database}", + username="not" + self.username, password="not" + self.password) except DatabaseError: # expected to fail, username and password incorrect pass # override username and password parameters from within url - self.attempt_connect(f"mapi:monetdb://{self.username}:{self.password}@{self.hostname}:{self.port}/{self.database}", username="not" + self.username, password="not" + self.password) + s = f"mapi:monetdb://{self.username}:{self.password}@{self.hostname}:{self.port}/{self.database}" + self.attempt_connect(s, username="not" + self.username, password="not" + self.password) def test_ipv4_address(self): try: @@ -53,7 +57,8 @@ def test_ipv4_address(self): except Exception: # can't test, return success return - self.attempt_connect(f"mapi:monetdb://{ip}:{self.port}/{self.database}", username=self.username, password=self.password) + self.attempt_connect(f"mapi:monetdb://{ip}:{self.port}/{self.database}", + username=self.username, password=self.password) def test_unix_domain_socket(self): sock_path = "/tmp/.s.monetdb.%i" % self.port diff --git a/tests/test_oid.py b/tests/test_oid.py index 577795e4..7acfe946 100644 --- a/tests/test_oid.py +++ b/tests/test_oid.py @@ -11,4 +11,4 @@ def setUp(self): def test_oid(self): q = "select tag from sys.queue()" self.cursor.execute(q) - rows = self.cursor.fetchall() + self.cursor.fetchall() diff --git a/tests/test_udf.py b/tests/test_udf.py index 63c75eeb..ec868c05 100644 --- a/tests/test_udf.py +++ b/tests/test_udf.py @@ -1,4 +1,4 @@ -from unittest import TestCase, SkipTest, skip +from unittest import TestCase, SkipTest import tempfile from typing import Optional from tests.util import test_args diff --git a/tests/util.py b/tests/util.py index 7c01e0df..d584707e 100644 --- a/tests/util.py +++ b/tests/util.py @@ -32,4 +32,3 @@ have_lz4 = True except ModuleNotFoundError: have_lz4 = False - diff --git a/tests/windows_tests.py b/tests/windows_tests.py index ff5cfe32..faed7f95 100644 --- a/tests/windows_tests.py +++ b/tests/windows_tests.py @@ -9,7 +9,7 @@ import pytest -def start_mserver(monetdbdir, farmdir, dbname, port, logfile): +def start_mserver(monetdbdir, farmdir, dbname, port, logfile): # noqa: C901 exe = os.path.join(monetdbdir, 'bin', 'mserver5') if platform.system() == 'Windows': exe += '.exe'