diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aca5df..519d25f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,166 +1,173 @@ # Changelog +## v3.6.5 (2024-09-19) + +### Bug Fixes + +- Fix(MacOS): Enhanced platform check for stackql installation +- Fix(Test Code): Removed loading of `test.env` in test execution script and Add mocking logic for `pystackql.StackQL`'s methods. + ## v3.6.4 (2024-07-17) ### Updates - * added dataflow dependency arguments +- added dataflow dependency arguments ## v3.6.3 (2024-06-22) ### Updates - * build updates +- build updates ## v3.6.2 (2024-05-06) ### Updates - * added `rowsaffected` to dict response for `executeStmt` +- added `rowsaffected` to dict response for `executeStmt` ## v3.6.1 (2024-04-18) ### Updates - * modified dict response for `executeStmt` - * modified error response for `execute`, should never return `None` +- modified dict response for `executeStmt` +- modified error response for `execute`, should never return `None` ## v3.5.4 (2024-04-11) ### Updates - * added `suppress_errors` argument to the `execute` function +- added `suppress_errors` argument to the `execute` function ## v3.5.3 (2024-04-08) ### Updates - * added `backend_storage_mode` and `backend_file_storage_location` constructor args for specifying a file based backend (not applicable in `server_mode`) +- added `backend_storage_mode` and `backend_file_storage_location` constructor args for specifying a file based backend (not applicable in `server_mode`) ## v3.5.2 (2024-03-21) ### Updates - * added `custom_registry` constructor arg for specifying a non-default registry +- added `custom_registry` constructor arg for specifying a non-default registry ## v3.5.1 (2024-03-15) ### Updates - * included `pandas` and `IPython` install requirements - * optional required import of `psycopg2` only if in `server_mode` +- included `pandas` and `IPython` install requirements +- optional required import of `psycopg2` only if in `server_mode` ## v3.2.5 (2023-12-07) ### Updates - * included `app_root` and `execution_concurrency_limit` options in `StackQL` constructor +- included `app_root` and `execution_concurrency_limit` options in `StackQL` constructor ## v3.2.4 (2023-10-24) ### Updates - * implemented non `server_mode` magic extension - * updated dataframe output for statements - * `pandas` type updates - * updated class parameters - * added additional tests - * bin path defaults for codespaces notebooks +- implemented non `server_mode` magic extension +- updated dataframe output for statements +- `pandas` type updates +- updated class parameters +- added additional tests +- bin path defaults for codespaces notebooks ## v3.0.0 (2023-10-11) ### Updates - * added `StackqlMagic` class for `jupyter`, `IPython` integration - * `server_mode` is now used to connect to a `stackql` server process using `pyscopg2` - * added additional tests +- added `StackqlMagic` class for `jupyter`, `IPython` integration +- `server_mode` is now used to connect to a `stackql` server process using `pyscopg2` +- added additional tests ## v2.0.0 (2023-08-15) ### Updates - * added `executeQueriesAsync` stackql class method +- added `executeQueriesAsync` stackql class method ## v1.5.0 (2023-04-04) ### Updates - * added `server_mode` to run a background stackql server process +- added `server_mode` to run a background stackql server process ## v1.0.2 (2023-02-23) ### Updates - * enabled custom `download_dir` argument +- enabled custom `download_dir` argument ## v1.0.1 (2023-02-23) ### Minor updates - * updated `setup.py` +- updated `setup.py` ## v1.0.0 (2023-02-22) ### Refactor - * refactored package - * added support for `kwargs` for the `StackQL` constructor - * added `setup.py` - * added `docs` using `sphinx` - * added additional tests +- refactored package +- added support for `kwargs` for the `StackQL` constructor +- added `setup.py` +- added `docs` using `sphinx` +- added additional tests ## v0.9.0 (2022-06-06) ### Bug Fixes - * added exception handling +- added exception handling ## v0.5.0 (2022-06-03) ### Bug Fixes - * added local registry support - * updated docs +- added local registry support +- updated docs ## v0.4.1 (2022-05-31) ### Bug Fixes - * added `str` handling - * updated docs +- added `str` handling +- updated docs ## v0.4.0 (2022-02-08) ### Updates - * updated `version` output - * updated docs +- updated `version` output +- updated docs ## v0.3.0 (2022-02-07) ### Initial release as `pystackql` - * added `auth` switch - * converted `byte` output to `str` +- added `auth` switch +- converted `byte` output to `str` ## v0.2.0 (2021-07-19) ### Updates to `pyinfraql` - * added `version` method - * updates to accept `None` for arguments - * updated docs +- added `version` method +- updates to accept `None` for arguments +- updated docs ## v0.1.1 (2021-07-16) ### Updates to `pyinfraql` - * added `dbfilepath` argument +- added `dbfilepath` argument ## v0.1.0 (2021-02-15) ### Initial Release (as `pyinfraql`) - * class constructor for `pyinfraql` - * support for `google` provider - * support for integration with `pandas`, `matplotlib` and `jupyter` \ No newline at end of file +- class constructor for `pyinfraql` +- support for `google` provider +- support for integration with `pandas`, `matplotlib` and `jupyter` diff --git a/docs/source/conf.py b/docs/source/conf.py index e435166..d7a5306 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ # The short X.Y version version = '' # The full version, including alpha/beta/rc tags -release = '3.6.4' +release = 'v3.6.5' # -- General configuration --------------------------------------------------- @@ -158,4 +158,4 @@ ] -# -- Extension configuration ------------------------------------------------- \ No newline at end of file +# -- Extension configuration ------------------------------------------------- diff --git a/pystackql/_util.py b/pystackql/_util.py index ea2d3bb..44bd488 100644 --- a/pystackql/_util.py +++ b/pystackql/_util.py @@ -80,25 +80,28 @@ def _download_file(url, path, showprogress=True): exit(1) def _setup(download_dir, platform, showprogress=False): - print('installing stackql...') - try: - binary_name = _get_binary_name(platform) - url = _get_url() - print("downloading latest version of stackql from %s to %s" % (url, download_dir)) - archive_file_name = os.path.join(download_dir, os.path.basename(url)) - _download_file(url, archive_file_name, showprogress) - if platform == 'Darwin': - unpacked_file_name = os.path.join(download_dir, 'stackql') - command = 'pkgutil --expand-full {} {}'.format(archive_file_name, unpacked_file_name) - os.system(command) - else: - with zipfile.ZipFile(archive_file_name, 'r') as zip_ref: - zip_ref.extractall(download_dir) - - os.chmod(os.path.join(download_dir, binary_name), 0o755) - except Exception as e: - print("ERROR: [_setup] %s" % (str(e))) - exit(1) + try: + print('installing stackql...') + binary_name = _get_binary_name(platform) + url = _get_url() + print("downloading latest version of stackql from %s to %s" % (url, download_dir)) + archive_file_name = os.path.join(download_dir, os.path.basename(url)) + _download_file(url, archive_file_name, showprogress) + # if Platform is starting with Darwin, then it is a MacOS + if platform.startswith('Darwin'): + unpacked_file_name = os.path.join(download_dir, 'stackql') + command = 'pkgutil --expand-full {} {}'.format(archive_file_name, unpacked_file_name) + # if there are files in unpacked_file_name, then remove them + if os.path.exists(unpacked_file_name): + os.system('rm -rf {}'.format(unpacked_file_name)) + os.system(command) + else: + with zipfile.ZipFile(archive_file_name, 'r') as zip_ref: + zip_ref.extractall(download_dir) + os.chmod(os.path.join(download_dir, binary_name), 0o755) + except Exception as e: + print("ERROR: [_setup] %s" % (str(e))) + exit(1) def _get_version(bin_path): try: diff --git a/run_tests b/run_tests index 44ae8bc..6fea2b4 100644 --- a/run_tests +++ b/run_tests @@ -1,3 +1,2 @@ #!/bin/bash -. tests/creds/env_vars/test.env -python3 -m tests.pystackql_tests \ No newline at end of file +python3 -m tests.pystackql_tests diff --git a/setup.py b/setup.py index 52be571..7716fe0 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='pystackql', - version='3.6.4', + version='v3.6.5', description='A Python interface for StackQL', long_description=readme, author='Jeffrey Aven', @@ -41,4 +41,4 @@ 'Programming Language :: Python :: 3.12', 'License :: OSI Approved :: MIT License', ] -) \ No newline at end of file +) diff --git a/tests/pystackql_tests.py b/tests/pystackql_tests.py index 28e7d16..0857d75 100644 --- a/tests/pystackql_tests.py +++ b/tests/pystackql_tests.py @@ -1,18 +1,18 @@ import sys, os, unittest, asyncio, re -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from pystackql import StackQL, magic, magics, StackqlMagic, StackqlServerMagic from .test_params import * def pystackql_test_setup(**kwargs): def decorator(func): - def wrapper(self): + def wrapper(self, *args): try: del self.stackql except AttributeError: pass self.stackql = StackQL(**kwargs) - func(self) + func(self, *args) return wrapper return decorator @@ -201,8 +201,14 @@ def test_11a_execute_with_defaults_null_response(self): print_test_result(f"Test execute with defaults (empty response)\nRESULT: {result}", is_valid_empty_resp) @pystackql_test_setup(output='pandas') - def test_12_execute_with_pandas_output(self): - # result = self.stackql.execute(aws_query) + @patch('pystackql.StackQL.execute') + def test_12_execute_with_pandas_output(self, mock_execute): + # mocking the response for pandas DataFrame + mock_execute.return_value = pd.DataFrame({ + 'status': ['RUNNING', 'TERMINATED'], + 'num_instances': [2, 1] + }) + result = self.stackql.execute(google_query) is_valid_dataframe = isinstance(result, pd.DataFrame) self.assertTrue(is_valid_dataframe, f"Result is not a valid DataFrame: {result}") @@ -221,8 +227,10 @@ def test_12_execute_with_pandas_output(self): print_test_result(f"Test execute with pandas output\nRESULT COUNT: {len(result)}", is_valid_dataframe) @pystackql_test_setup(output='csv') - def test_13_execute_with_csv_output(self): - # result = self.stackql.execute(aws_query) + @patch('pystackql.StackQL.execute') + def test_13_execute_with_csv_output(self, mock_execute): + # mocking the response for csv output + mock_execute.return_value = "status,num_instances\nRUNNING,2\nTERMINATED,1\n" result = self.stackql.execute(google_query) is_valid_csv = isinstance(result, str) and result.count("\n") >= 1 and result.count(",") >= 1 self.assertTrue(is_valid_csv, f"Result is not a valid CSV: {result}") @@ -280,22 +288,34 @@ def test_20a_executeStmt_server_mode_with_pandas_output(self): print_test_result(f"Test executeStmt in server mode with pandas output\n{result_df}", is_valid_response, True) @pystackql_test_setup(server_mode=True) - def test_21_execute_server_mode_default_output(self): + @patch('pystackql.stackql.StackQL._run_server_query') + def test_21_execute_server_mode_default_output(self, mock_run_server_query): + # Mocking the response as a list of dictionaries + mock_result = [ + {'status': 'RUNNING', 'num_instances': 2}, + {'status': 'TERMINATED', 'num_instances': 1} + ] + mock_run_server_query.return_value = mock_result + result = self.stackql.execute(google_query) is_valid_dict_output = isinstance(result, list) and all(isinstance(row, dict) for row in result) print_test_result(f"""Test execute in server_mode with default output\nRESULT_COUNT: {len(result)}""", is_valid_dict_output, True) + # Check `_run_server_query` method + mock_run_server_query.assert_called_once_with(google_query) @pystackql_test_setup(server_mode=True, output='pandas') - def test_22_execute_server_mode_pandas_output(self): - # result = self.stackql.execute(aws_query) + @patch('pystackql.stackql.StackQL._run_server_query') + def test_22_execute_server_mode_pandas_output(self, mock_run_server_query): + # Mocking the response for pandas DataFrame + mock_df = pd.DataFrame({ + 'status': ['RUNNING', 'TERMINATED'], + 'num_instances': [2, 1] + }) + mock_run_server_query.return_value = mock_df.to_dict(orient='records') result = self.stackql.execute(google_query) is_valid_dataframe = isinstance(result, pd.DataFrame) self.assertTrue(is_valid_dataframe, f"Result is not a valid DataFrame: {result}") # Check datatypes of the columns - # expected_dtypes = { - # 'instance_type': 'object', - # 'num_instances': 'int64' - # } expected_dtypes = { 'status': 'object', 'num_instances': 'int64' @@ -304,6 +324,8 @@ def test_22_execute_server_mode_pandas_output(self): actual_dtype = result[col].dtype self.assertEqual(actual_dtype, expected_dtype, f"Column '{col}' has dtype '{actual_dtype}' but expected '{expected_dtype}'") print_test_result(f"Test execute in server_mode with pandas output\nRESULT COUNT: {len(result)}", is_valid_dataframe) + # Check `_run_server_query` method + mock_run_server_query.assert_called_once_with(google_query) class MockInteractiveShell: """A mock class for IPython's InteractiveShell.""" diff --git a/tests/test_params.py b/tests/test_params.py index 4fbb0dc..ed0b662 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -1,4 +1,4 @@ -import json, os, sys, platform, re, time, unittest, subprocess +import json, sys, platform, re, time, unittest, subprocess import pandas as pd from termcolor import colored import unittest.mock as mock @@ -33,11 +33,14 @@ def get_custom_download_dir(platform_name): def registry_pull_resp_pattern(provider): return r"%s provider, version 'v\d+\.\d+\.\d+' successfully installed\s*" % provider +test_gcp_project_id = "test-gcp-project" +test_gcp_zone = "australia-southeast2-a" + google_query = f""" SELECT status, count(*) as num_instances FROM google.compute.instances -WHERE project = '{os.environ['GCP_PROJECT']}' -AND zone = '{os.environ['GCP_ZONE']}' +WHERE project = '{test_gcp_project_id}' +AND zone = '{test_gcp_zone}' GROUP BY status """ @@ -50,7 +53,7 @@ def registry_pull_resp_pattern(provider): GROUP BY instance_type """ -regions = os.environ.get('AWS_REGIONS').split(',') +test_aws_regions = ["ap-southeast-2", "ap-southeast-4"] async_queries = [ f""" @@ -58,7 +61,7 @@ def registry_pull_resp_pattern(provider): FROM aws.lambda.functions WHERE region = '{region}' """ - for region in regions + for region in test_aws_regions ] def print_test_result(test_name, condition=True, server_mode=False, is_ipython=False): @@ -73,4 +76,4 @@ def print_test_result(test_name, condition=True, server_mode=False, is_ipython=F headers.append(test_name) message = " ".join(headers) - print("\n" + message) \ No newline at end of file + print("\n" + message)