diff --git a/integration/keeper_secrets_manager_cli/README.md b/integration/keeper_secrets_manager_cli/README.md index e5c65b90..ebe34716 100644 --- a/integration/keeper_secrets_manager_cli/README.md +++ b/integration/keeper_secrets_manager_cli/README.md @@ -6,6 +6,11 @@ For more information see our official documentation page https://docs.keeper.io/ # Change History +## 1.0.16 + +* KSM-362 - Synchronize secrets to GCP +* Dropped support for Python 3.6 (EOL 2021-12-23) + ## 1.0.15 * Update pinned KSM SDK version. The KSM SDK has been updated to use OpenSSL 3.0.7 which fixes CVE-2022-3602, CVE-2022-3786. diff --git a/integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/__main__.py b/integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/__main__.py index 64c2185d..fbf8087a 100644 --- a/integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/__main__.py +++ b/integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/__main__.py @@ -117,7 +117,7 @@ def get_command(self, ctx, cmd_name): if best_score > 0.50: cmd_name = best_command - return super().get_command(ctx, cmd_name) + return super().get_command(ctx, str(cmd_name)) def parse_args(self, ctx, args: t.List[str]): @@ -194,8 +194,8 @@ def base_command_help(f): doc = f.__doc__ versions = get_versions() - cli_version = versions.get("keeper-secrets-manager-cli") - sdk_version = versions.get("keeper-secrets-manager-core") + cli_version = versions.get("keeper-secrets-manager-cli", "") + sdk_version = versions.get("keeper-secrets-manager-core", "") doc = "{} Version: {} ".format( Fore.RED + doc + Style.RESET_ALL, @@ -1047,7 +1047,7 @@ def shell_command(app): versions = get_versions() - print(Fore.CYAN + "Current Version: " + Fore.GREEN + versions.get("keeper-secrets-manager-cli") + Style.RESET_ALL) + print(Fore.CYAN + "Current Version: " + Fore.GREEN + versions.get("keeper-secrets-manager-cli", "") + Style.RESET_ALL) update = update_available("keeper-secrets-manager-cli", versions) if update is not None: print(Fore.YELLOW + "Version {} is available.".format(update.available_version) + Style.RESET_ALL) @@ -1080,9 +1080,9 @@ def quit_command(): @click.option('--credentials', '-c', type=str, metavar="UID", help="Keeper record with credentials to access destination key/value store.", cls=Mutex, # not_required_if=[('type','json')], - required_if=[('type','azure'), ('type','aws')] + required_if=[('type','azure'), ('type','aws'), ('type','gcp')] ) -@click.option('--type', '-t', type=click.Choice(['aws', 'azure', 'json']), default='json', help="Type of the target key/value storage (aws, azure, json).", show_default=True) +@click.option('--type', '-t', type=click.Choice(['aws', 'azure', 'gcp', 'json']), default='json', help="Type of the target key/value storage (aws, azure, gcp, json).", show_default=True) @click.option('--dry-run', '-n', is_flag=True, help='Perform a trial run with no changes made.') @click.option('--preserve-missing', '-p', is_flag=True, help='Preserve destination value when source value is deleted.') @click.option('--map', '-m', nargs=2, type=(str, str), multiple=True, required=True, metavar="...", help='Map destination key names to values using notation URI.') diff --git a/integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/sync.py b/integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/sync.py index 343e60d2..d1c13b0a 100644 --- a/integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/sync.py +++ b/integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/sync.py @@ -30,6 +30,9 @@ AWS_SECRET_ACCESS_KEY_LABEL = "AWS Secret Access Key" AWS_REGION_NAME_LABEL = "AWS Region Name" +GOOGLE_CLOUD_PROJECT_ID_LABEL = "Google Cloud Project ID" +GOOGLE_APPLICATION_CREDENTIALS_LABEL = "Google Application Credentials" # If missing use default creds (ADC) + class Sync: def __init__(self, cli): @@ -41,7 +44,7 @@ def __init__(self, cli): self.local_cache = {} def _output(self, data: list, hide_data:bool=False): - data = data or {} + data = data or [] failed = sum(1 for x in data if x.get("error", "") != "") output = { "data": data, @@ -555,11 +558,176 @@ def _delete_secret_aws(self, client, key): result["error"] = str(e) return result - def sync_values(self, type:str, credentials:str=None, dry_run=False, preserve_missing=False, map=None): + def _get_secret_gcp(self, client, project_id, secret_id): + from google.api_core.exceptions import ( + ClientError, + GoogleAPIError, + NotFound, + ServerError + ) + + result = { + "value": None, + "not_found": False, + "error": None + } + + try: + version_id="latest" + name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" + secret = client.access_secret_version(request={"name": name}) + result["value"] = secret.payload.data.decode("UTF-8") + # crc32c = google_crc32c.Checksum() + # crc32c.update(secret.payload.data) + # if secret.payload.data_crc32c != int(crc32c.hexdigest(), 16): + # result["error"] = f"Data corruption detected for key={secret_id}" + except NotFound as e: + # Deleted or non existing key. + self.logger.debug(f"GCP Client: secret not found. key={secret_id} Message: {e.message}") + result["not_found"] = True + except ClientError as e: + # Includes - PermissionDenied, Forbidden, Unauthenticated, Unauthorized + self.log.append(f"GCP SDK Client error. {e.message} Skipping key={secret_id}") + self.logger.error("GCP SDK Client error. " + str(e)) + result["error"] = str(e) + except ServerError as e: + self.log.append(f"GCP SDK Server error. {e.message} Skipping key={secret_id}") + self.logger.error("GCP SDK Server error. " + str(e)) + result["error"] = str(e) + except GoogleAPIError as e: + # Will catch everything that is from GCP SDK + self.log.append(f"GCP API error. {str(e)} Skipping key={secret_id}") + self.logger.error("GCP API error. " + str(e)) + result["error"] = str(e) + except Exception as e: + self.log.append(f"Error retrieving secret. Skipping key={secret_id}") + self.logger.error("Error retrieving secret. " + str(e)) + result["error"] = str(e) + return result + + def _set_secret_gcp(self, client, project_id:str, secret_id:str, value:str): + from google.api_core.exceptions import ( + AlreadyExists, + ClientError, + GoogleAPIError, + NotFound, + ServerError + ) + + result = { + "success": False, + "error": None + } + + existing_value = None + try: + version_id="latest" + name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" + secret = client.access_secret_version(request={"name": name}) + existing_value = secret.payload.data.decode("UTF-8") + except Exception: + pass + + # avoid creating new versions with the same value + if existing_value == value: + result["success"] = True + return result + + err = "" + if existing_value is None: # key doesn't exist - create + try: + parent = f"projects/{project_id}" + secret = client.create_secret( + request={ + "parent": parent, + "secret_id": secret_id, + "secret": {"replication": {"automatic": {}}}, + } + ) + if not(secret and secret.create_time): + err = f"Failed to create secret {secret_id} " + except AlreadyExists as e: + pass + except Exception as e: + err = f"Failed to create secret {secret_id} - Error: {str(e)}" + + try: + parent = client.secret_path(project_id, secret_id) + response = client.add_secret_version(request={"parent": parent, "payload": {"data": value.encode("UTF-8")}}) + if response and response.create_time: + result["success"] = True + except NotFound as e: + # Deleted or non existing key. + self.logger.debug(f"GCP Client: secret not found. key={secret_id} Message: {e.message}") + result["error"] = "Key not found. Error: " + err + str(e) + except ClientError as e: + # Includes - PermissionDenied, Forbidden, Unauthenticated, Unauthorized + self.log.append(f"GCP SDK Client error. {e.message} Skipping key={secret_id}") + self.logger.error("GCP SDK Client error. " + str(e)) + result["error"] = err + str(e) + except ServerError as e: + self.log.append(f"GCP SDK Server error. {e.message} Skipping key={secret_id}") + self.logger.error("GCP SDK Server error. " + str(e)) + result["error"] = err + str(e) + except GoogleAPIError as e: + # Will catch everything that is from GCP SDK + self.log.append(f"GCP API error. {str(e)} Skipping key={secret_id}") + self.logger.error("GCP API error. " + str(e)) + result["error"] = err + str(e) + except Exception as e: + self.log.append(f"Unknown error. Skipping key={secret_id}") + self.logger.error("Unknown error. " + str(e) + " : " + err) + result["error"] = err + str(e) + return result + + def _delete_secret_gcp(self, client, project_id:str, secret_id:str): + from google.api_core.exceptions import ( + ClientError, + GoogleAPIError, + NotFound, + ServerError + ) + + result = { + "success": False, + "error": None + } + + try: + # Delete the secret with the given name and all of its versions. + name = client.secret_path(project_id, secret_id) + client.delete_secret(request={"name": name}) + result["success"] = True + except NotFound as e: + # Deleted or non existing key. + self.logger.debug(f"GCP Client Error: NotFound while trying to delete secret. key={secret_id} already deleted. Message: {e.message}") + result["success"] = True # already deleted + result["error"] = str(e) + except ClientError as e: + # Includes - PermissionDenied, Forbidden, Unauthenticated, Unauthorized + self.log.append(f"GCP SDK Client error. {e.message} Skipping key={secret_id}") + self.logger.error("GCP SDK Client error. " + str(e)) + result["error"] = str(e) + except ServerError as e: + self.log.append(f"GCP SDK Server error. {e.message} Skipping key={secret_id}") + self.logger.error("GCP SDK Server error. " + str(e)) + result["error"] = str(e) + except GoogleAPIError as e: + # Will catch everything that is from GCP SDK + self.log.append(f"GCP API error. {str(e)} Skipping key={secret_id}") + self.logger.error("GCP API error. " + str(e)) + result["error"] = str(e) + except Exception as e: + self.log.append(f"Error deleting secret. Skipped delete key={secret_id}") + self.logger.error("Error deleting secret. " + str(e)) + result["error"] = str(e) + return result + + def sync_values(self, type:str, credentials:str="", dry_run=False, preserve_missing=False, map=None): map = map or [] result = [] - """ + r""" stats = { "totalMappings": 0, "badMappings": [], # bad notation @@ -617,10 +785,12 @@ def sync_values(self, type:str, credentials:str=None, dry_run=False, preserve_mi self.sync_azure(credentials, dry_run, preserve_missing, result) elif type == 'aws': self.sync_aws(credentials, dry_run, preserve_missing, result) + elif type == 'gcp': + self.sync_gcp(credentials, dry_run, preserve_missing, result) else: - raise KsmCliException(f"Invalid option `--type {type}`. Allowed values are (json, azure, aws).") + raise KsmCliException(f"Invalid option `--type {type}`. Allowed values are (json, azure, aws, gcp).") - def sync_azure(self, credentials:str=None, dry_run=False, preserve_missing=False, map:dict=None): + def sync_azure(self, credentials:str="", dry_run=False, preserve_missing=False, map:list=[]): try: from azure.keyvault.secrets import SecretClient from azure.identity import ClientSecretCredential @@ -689,7 +859,7 @@ def sync_azure(self, credentials:str=None, dry_run=False, preserve_missing=False err_msg = res.get("error", "") if err_msg: if "(SecretNotFound)" in err_msg: - self.logger.debug("Failed to delete key=" + key) # already deleted + self.logger.debug(f"Failed to delete key={key} - Already deleted.") # already deleted else: m["error"] = "Failed to delete remote key value pair." self.log.append(f"Failed to delete key={key}") @@ -702,7 +872,7 @@ def sync_azure(self, credentials:str=None, dry_run=False, preserve_missing=False self.logger.error("Failed to set new value for key=" + key) self._output(map, True) - def sync_aws(self, credentials:str=None, dry_run=False, preserve_missing=False, map:dict=None): + def sync_aws(self, credentials:str="", dry_run=False, preserve_missing=False, map:list=[]): try: import boto3 except ImportError as ie: @@ -779,3 +949,85 @@ def sync_aws(self, credentials:str=None, dry_run=False, preserve_missing=False, self.log.append(f"Failed to set new value for key={key}") self.logger.error("Failed to set new value for key=" + key) self._output(map, True) + + def sync_gcp(self, credentials:str="", dry_run=False, preserve_missing=False, map:list=[]): + try: + from google.cloud import secretmanager + from google.oauth2 import service_account + except ImportError as ie: + print(Fore.RED + "Missing GCP dependencies. To install missing packages run: \r\n" + + Fore.YELLOW + "pip3 install --upgrade google-cloud-secret-manager google-auth\r\n" + Style.RESET_ALL, file=sys.stderr) + raise KsmCliException("Missing GCP Dependencies: " + str(ie)) + + if not map or len(map) == 0: + print(Fore.YELLOW + "Nothing to sync - please provide some values with `--map \"key\" \"value\"`" + Style.RESET_ALL, file=sys.stderr) + return + + if not credentials or not str(credentials).strip(): + print(Fore.YELLOW + "Missing credentials' record UID - please provide UID with `--credentials `" + Style.RESET_ALL, file=sys.stderr) + return + + credentials = str(credentials).strip() + secrets = self.cli.client.get_secrets(uids=[credentials]) + if len(secrets) == 0: + raise KsmCliException("Cannot find the record with GCP credentials " + credentials) + creds = secrets[0] + + # NB! Labels are case sensitive. Use Hidden Field fields in custom section of the record. + app_credentials = self._get_secret_field(creds, GOOGLE_APPLICATION_CREDENTIALS_LABEL) or "" + project_id = self._get_secret_field(creds, GOOGLE_CLOUD_PROJECT_ID_LABEL) + + if not project_id: + print(Fore.YELLOW + "Missing Project Id in credentials record " + credentials + Style.RESET_ALL, file=sys.stderr) + raise KsmCliException(f"Cannot find all required credentials in record UID {credentials}.") + + # If credentials are provided, the corresponding JSON is used first, then it defaults to ADC + # If credentials are empty GCP client will use Application Default Credentials (ADC) + # ADC can be acquired by running `gcloud auth application-default login` on same host + # To specify non-default credentials location set env var: GOOGLE_APPLICATION_CREDENTIALS="/path/to/credentials.json" + # https://cloud.google.com/docs/authentication/provide-credentials-adc + + client:secretmanager.SecretManagerServiceClient|None = None + if str(app_credentials).strip(): + gcp_json_credentials_dict = json.loads(app_credentials) + credentialz = service_account.Credentials.from_service_account_info(gcp_json_credentials_dict) + client = secretmanager.SecretManagerServiceClient(credentials=credentialz) + + if client is None: + client = secretmanager.SecretManagerServiceClient() + + if dry_run: + for m in map: + key = m["mapKey"] + res = self._get_secret_gcp(client, project_id, key) + val = res.get("value", None) + m["dstValue"] = val if val else None + if not res.get("not_found", False) and res.get("error", ""): + self.log.append(f"Error reading the value from GCP for key={key}") + self._output(map) + else: + for m in map: + key = m["mapKey"] + val = m["srcValue"] + m["dstValue"] = m["srcValue"] + if val is None: + if preserve_missing: + continue + else: + res = self._delete_secret_gcp(client, project_id, key) + err_msg = res.get("error", "") + if err_msg: + # '404 Secret [projects/123456789012/secrets/key_name] not found.' + if err_msg.startswith('404 ') and err_msg.endswith(' not found.'): + self.logger.debug(f"Failed to delete key={key} - Already deleted.") # already deleted + else: + m["error"] = "Failed to delete remote key value pair." + self.log.append(f"Failed to delete key={key}") + self.logger.error("Failed to delete key=" + key) + else: + res = self._set_secret_gcp(client, project_id, key, val) + if res.get("error", ""): + m["error"] = "Failed to set new value for the key." + self.log.append(f"Failed to set new value for key={key}") + self.logger.error("Failed to set new value for key=" + key) + self._output(map, True) diff --git a/integration/keeper_secrets_manager_cli/requirements.txt b/integration/keeper_secrets_manager_cli/requirements.txt index 3377cc37..8e9d038e 100644 --- a/integration/keeper_secrets_manager_cli/requirements.txt +++ b/integration/keeper_secrets_manager_cli/requirements.txt @@ -1,4 +1,4 @@ -keeper-secrets-manager-core>=16.4.1 +keeper-secrets-manager-core>=16.5.1 keeper-secrets-manager-helper prompt-toolkit~=2.0 jsonpath-rw-ext diff --git a/integration/keeper_secrets_manager_cli/setup.py b/integration/keeper_secrets_manager_cli/setup.py index 6b66991f..fece0206 100644 --- a/integration/keeper_secrets_manager_cli/setup.py +++ b/integration/keeper_secrets_manager_cli/setup.py @@ -8,7 +8,7 @@ long_description = f.read() install_requires = [ - 'keeper-secrets-manager-core>=16.4.1', + 'keeper-secrets-manager-core>=16.5.1', 'keeper-secrets-manager-helper', 'prompt-toolkit~=2.0', 'click', @@ -25,19 +25,19 @@ # Version set in the keeper_secrets_manager_cli.version file. setup( name="keeper-secrets-manager-cli", - version="1.0.15", + version="1.0.16", description="Command line tool for Keeper Secrets Manager", long_description=long_description, long_description_content_type="text/markdown", author="Keeper Security", - author_email="ops@keepersecurity.com", + author_email="sm@keepersecurity.com", url="https://github.com/Keeper-Security/secrets-manager", license="MIT", keywords="Keeper Password Secrets Manager SDK CLI", packages=find_packages(exclude=["tests", "tests.*"]), zip_safe=False, install_requires=install_requires, - python_requires='>=3.6', + python_requires='>=3.7', project_urls={ "Bug Tracker": "https://github.com/Keeper-Security/secrets-manager/issues", "Documentation": "https://app.gitbook.com/" @@ -51,7 +51,6 @@ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/integration/keeper_secrets_manager_cli/tests/exec_test.py b/integration/keeper_secrets_manager_cli/tests/exec_test.py index 8662b17b..98c033f0 100644 --- a/integration/keeper_secrets_manager_cli/tests/exec_test.py +++ b/integration/keeper_secrets_manager_cli/tests/exec_test.py @@ -197,12 +197,19 @@ def test_cmd_bad(self): Profile.init(token='MY_TOKEN') # Make a temp shell script - with tempfile.NamedTemporaryFile(delete=False) as script: + with tempfile.NamedTemporaryFile(delete=False, suffix=".BAT") as script: self.delete_me.append(script.name) - the_script = [ - "#!/bin/sh", - "echo 'MOOT'" - ] + if platform == "win32": + the_script = [ + "@echo off", + "setlocal enableDelayedExpansion", + "echo 'MOOT'" + ] + else: + the_script = [ + "#!/bin/sh", + "echo 'MOOT'" + ] script.write("\n".join(the_script).encode()) script.close() os.chmod(script.name, 0o777)