diff --git a/src/TA-egnyte-protect/README.md b/TA-EGNYTE-PROTECT-README.md similarity index 100% rename from src/TA-egnyte-protect/README.md rename to TA-EGNYTE-PROTECT-README.md diff --git a/src/TA-egnyte-protect/TA-egnyte-protect.aob_meta b/src/TA-egnyte-protect/TA-egnyte-protect.aob_meta new file mode 100644 index 0000000..d27ad61 --- /dev/null +++ b/src/TA-egnyte-protect/TA-egnyte-protect.aob_meta @@ -0,0 +1 @@ +{"basic_builder": {"appname": "TA-egnyte-protect", "friendly_name": "Egnyte Secure and Govern Add-on For Splunk", "version": "1.1.1", "author": "", "description": "", "theme": "#65A637", "large_icon": "iVBORw0KGgoAAAANSUhEUgAAAEgAAABICAYAAABV7bNHAAAAAXNSR0IArs4c6QAACHhJREFUeF7t3Idu3LwSBWA5vffee/L+TxMgvffeq398uvcYtKLtq7UNiMDC2RVFcc6cOTOkiCxVfRuKwFKPz3AEeoBGMKQHqAdoNhHpGTQtg27evLk8G/Yb6+4bN260kmUgg3qA/ufgHqD/E71n0IiI7wHqAZotKfQM2sgM2rFjR7Vnz54VE5aXl6vPnz9XP378mI0WE9y9rhl06tSpCkg/f/6sTdq2bVv15cuX6sWLFxOYOFvXNQVo06ZN1f79+2tW/Pr1a5UlS0tL1blz56qPHz9W7969q68dOXKkBuzJkycVNpUNeLt27ao+fPjwz7VZIFozgPbt21cdPXq02rx5cx0yDx8+rLZv317t3bu3NtS/gfTo0aPq69evtY2uYdXfv3/re/z+6dOnGtzz589XW7durX7//l29fPmyBn0ebeEAAeT06dM1AG/evKk9zji/Y5QQYvj3799rEP78+bPKTv2wyP27d++uwQQYkIB58ODB6tChQ/UYT58+ra/N0hYOEC9funSpevz4cQ1GtIWh2NAEZJRxW7ZsqYXcWAlTTDt58mR1+/btjQcQgzEIEM+fPx9l/1TX5zn+whkkNM6cOVOHwLNnz/4BgO7s3Lmz/uib0CPKQJXRhJ/7B7HN+EQbS5MBp0K6qqqFAiSMeFdmIqRlJmIQ/RAeQAEC44QNHaFPftePBgFSWMlwEfGA4JoQ8zwgGWva1ilA8T4jsUHafvv2bS3OafrIZtI9Q96/f19r0TBxBQDj3QPQb9++1bVRs4A8duxYJVvKkMCmf1g3iXB3ChBAhIqUy+v+lkUeI3nahDEqoj2JtzEKEMZ69erVSs2UMcpiU9YzB3XUuK0zgHju+PHjdarlZV4nygkr3ncdW/K7PoyVlbDB7+qasunjPsaWfQ4cOFDfK3w5Ic/RH0jGAY4QB9C4zugEIHpx8eLF2ptCqtmAhzk8Xl6nQYcPH67ZxGBGAbhsahz9Xr9+XYdY2ScaF9DbWCKcOeD+/ftjVdydAMQAE7lz584/mUbInT17tgaGkWUDGs9jDuOBdffu3Yn6AEkWM3abc1KHyaDYNqp1ApBJoDUBLlMtZl24cKEW4yYzTLQJkLUXkMuWpcYwEBNuxLnMYHRQiEkawmycorQTgBgk9q9cuVKHkcykYZXwunfvXmsmmRdAKUZV2Q8ePFjBFyOBh5XNxe4gJnUGEG9ZYyn3eQqbLl++XIfPIGrPEyAspoOYmoWrjOc3ADV3DxYOELaofYSYxnt0BXvavIdxJ06cWKVBQqxNg5QFZYgNGtN4gLKITeM0gJW12DAd6oxBJiKbRCh5zvdSmE2eEYQ1TThK08l0bZOXHWW6Zh/FoHtTWUsIajE6lnKh6bg1EWlsuHr1as0ek03mkFrLahdoJs7gVLfZ4jBGlhRNIwb1UW9hKdYFkOigbRVNH3WU0B+nzZVBDFKnAMS/oz8mhSm3bt1amRMBpUlN0MaZ9LA+HINF0TlZC1ip4KNDWesJtWEL2rkCRDPQHiOy4xf9UZyVGUXKZ0y5YzgrOE3mJnNyVqlDqnBAcSYJyJZu2/PnDpDJRJjzQEsArGqugYQY6rcVdNOAFc0Jc42haJXaCXmz0clyz3vNABLz0nyzOJTWeb2taJwGIGD4lJkPOJjSzIbGXzhASeW8IsRkrUE0j3ebBk0DTO5pAxw4NLAMbzIgxIAnq6aQ7ZxB6h4PzSuYUN0kTYogl60tJGYBqC1khXd2F40dkbaaJ870Z1jROFcNinFZZggdEyHQ1lCyWFkk6nft2rVVG/jTAjRI9C2MsVndpOV1U3ONN+i5nQDkYQo0FSsBHpaxLF6F5KxCHTZyQmqqZLVyuaFQxKJxN806A0jKN+mkV4JoazSejMfURwCcVajbspV0rg4qt104ROYcltpLNnUGED0ymZT5Klx64Hu5Jzwsy0wSbgRaK9+U2BeyUM5vmdO6WKxihUo52x2+e2Goci2911z1TwJK2ZdAy0YZO2DYE8Jcbd1sd4hx1CbIYj1ZQhgIPUVb1kp0ApCj0u0w4AJGuWyhgdhTas2geQ0bu5MQy45e25YrLTLxstoGmnvyemYSFmGmTAXwgKGswBaANVN4Fs40L3XawgHCCpSXnZr7zryYCjYLSP0ZafL0ovkicJABxiLyqnTgAj7v5QdtzKnqhXVZOC4cIA9M7cNgtQcQcq4nG+s0I1kNE0xeX3WLD++XAGeBaazsGGCBMUpw6Fw2xIybFb2aDKAya3RpFFs7CbE8VBYBBib5W27WS8GKRxMFYjbQ6Qlwc1aorGvyWsh4wMvBK4AJKZ/yVZLfgWNrBeDYBdC2MwELLxQ9kPc0aZ33c2IsrIloChEex6hU2slu5W4gbcm2bQwCfBbDQja6ki1c14UTjfIccxl3w94zOmVQ0yuMLk92mChDwgyTV8T5RMfKmgVDsqYDFGHHNP0xJyzkGOx0TTjNcsJjoQBlsQgklC9TMO8ynuGaLNfccZTthAkBBoBQA04TAIKfd3LNV9ejNKd5feEAmQCAaMig06rYk3VUWegFoLbNr9Kwtqp6UmDSf+EApQ7BnryvEnpEm36UbLh+/fqqLdls6ZabXxH18l5MlK3a6rBJgVo4QPSBhwFiWSBEhIQm2+T0mL/6lStxAAlBepNDnP4KI/olLCPk5amRSUEp+y8coDwcQNnMkuqJKYB43y4AVtERGa88J02oARIgsdB3xSew/O6eceucUeCtGUAmRmeIrgKu7SBB8+VjDpI3XwoYqywcRxk9yfU1BWjURHPOMJkICDnfOOreeV1f1wApJIVimrqpPA89LxCGjbOuAVoEAKOe0QM0AqEeoB6gUUE0/HrPoJ5BPYNmQ2DeDOp0Nhto8P5/fxnhrB6gHqDZ4rlnUM+g2Rj0H1dT7aMuXxb/AAAAAElFTkSuQmCC", "small_icon": "iVBORw0KGgoAAAANSUhEUgAAACQAAAAkCAYAAADhAJiYAAAAAXNSR0IArs4c6QAAAxJJREFUWEftmIlOIlEQRQtxQVwQcEk0uIAC//81gIioEeK+ICBqYHIqU+QJCE0PcYjxJSa2TXfduksVMSBTdgJThkemG1Aul+v8D8ay2WyXmE8M/QL6K8fPZmhxcVE6nY7MzMzIx8eHvL29jW1D3wwFg0FJpVLSarWk0WhILBaTWq0mlUpFQezs7Mjy8rLc3d0JQPkpFovSbreHgvQNaG1tTZ6enpQRLwfWlpaWFPSw4xvQ4eGhnJycKKBAICBbW1sSiUS6tWDi+vpaQXMAdHBwoM9MDBAyraysyPr6utLP2dvbk3q9Lre3t311AAoIgL28vOh9ZL6/v5fHx8eB7I7FUDqdluPjY30xzGQyGcnn83p9dHSkRj49PdXr7e1tCYfDUiqVVCr8VK1WlSnY4/P2LreTsQDBxtnZWReAvXB+fl4BJhIJBcAxsMiIbDDLMQ/t7+9LuVzuY3UsQBsbG3Jzc6PdEmmLtRdAVEY+Y5BUIl3vGQuQde1KFY1GBUC8HIaen5+1RjweVzmJf7PZ1Pt4inQ+PDx8ktuXZMwRijNnNjc31agwhpm9RH93d1fOz88FHxYKBQUP26+vr59I8sxQKBTShzHlwsKCdg1AuvVyMDnNGCCewewk1BdDJIaJTPRnZ2d1QgMOoPx92HE/x/yykYHRewelZ4bwCRJdXl7qALSBZ50PA+Qyaf5DQsbA+/u7P4bcKCeTyb54DwPkpguT05QbDF+S8RATGhO7rHw1T9wiVhymbEJ/5T/PklGAZFxcXGgt6xpvMXl702KA8BwSE3uXKXfI+maIhJgh8RSmJLr4AWPTPeuDMzc3pykCDNPd0gVAGsDc/7w67CsEc4itTcLY8rZOXAbN+CQSzzF7jFm+I5GuQfNrLMlcatnarAG6BQiFkY3kcGAPr11dXSlzMAOTtjq+CoFvQINMabuOYqurq901wjVrA5C2WiYOiAL4gDlCEcDwu217ZMJHrBjA4Tn8N2rN+GZoUId4CmnwGvL1Dj0vK2aigLwUHPWZX0A/h6FRnXzH/en+/9B3MDCqxh9YRC5DRODzCQAAAABJRU5ErkJggg==", "visible": false, "tab_version": "4.1.3", "tab_build_no": "0", "build_no": 1}} \ No newline at end of file diff --git a/src/TA-egnyte-protect/app.manifest b/src/TA-egnyte-protect/app.manifest index 20fc0e0..43a637c 100644 --- a/src/TA-egnyte-protect/app.manifest +++ b/src/TA-egnyte-protect/app.manifest @@ -5,7 +5,7 @@ "id": { "group": null, "name": "TA-egnyte-protect", - "version": "1.1.0" + "version": "1.1.1" }, "author": [ { diff --git a/src/TA-egnyte-protect/bin/input_module_egnyte.py b/src/TA-egnyte-protect/bin/input_module_egnyte.py index 161dd33..1f4f713 100755 --- a/src/TA-egnyte-protect/bin/input_module_egnyte.py +++ b/src/TA-egnyte-protect/bin/input_module_egnyte.py @@ -2,29 +2,31 @@ import os import sys import time -import datetime import json from requests.exceptions import HTTPError -from solnlib.splunkenv import get_splunkd_uri -from solnlib.credentials import (CredentialManager, CredentialNotExistException) -import requests from ta_egnyte_protect_utility import * import ta_egnyte_constants as tec import splunk.rest as rest +import splunklib.client as client + APP_NAME = os.path.abspath(__file__).split(os.sep)[-3] + def validate_input(helper, definition): interval = float(definition.parameters.get('interval', None)) if interval < 600: helper.log_error("Interval must be at least 600 seconds.") raise Exception('Interval must be at least 600 seconds') + def get_checkpoint(helper, stanza_name): return helper.get_check_point(stanza_name) + def set_checkpoint(helper, stanza_name, state): return helper.save_check_point(stanza_name, state) + def collect_events(helper, ew): # getting setup parameters input_name = helper.get_input_stanza_names() @@ -42,34 +44,49 @@ def collect_events(helper, ew): if endpoint == "US": base_url = tec.us_url else: - base_url =tec.europe_url + base_url = tec.europe_url auth_url = str(base_url) + "/oauth2/token" - + checkpoint = get_checkpoint(helper, stanza_name) or dict() + + service = client.connect(host='localhost', port=8089, token=session_key) + # Going to take access/refresh token if it is not available in the checkpoint if not checkpoint or str(checkpoint.get("code")) != str(code): - helper.log_info("Checkpoint is not available or code changed from setup page. Hence requesting new access token.") + helper.log_info( + "Checkpoint is not available or code changed from setup page. Hence requesting new access token.") state = get_checkpoint(helper, stanza_name) or dict() try: - response = generate_or_refresh_token(helper=helper, auth_url=auth_url, clientid=clientid, client_secret=client_secret, code=code) - helper.log_info("Checkpoint is not available or code changed from setup page. Hence requested new access token.") + response = generate_or_refresh_token(helper=helper, auth_url=auth_url, clientid=clientid, + client_secret=client_secret, code=code) + helper.log_info( + "Checkpoint is not available or code changed from setup page. Hence requested new access token.") response = response.json() if response.get("error"): - helper.log_error("Error while getting access/refresh token error: {} error_description:{}".format(response.get("error", ""), response.get("error_description", ""))) + helper.log_error("Error while getting access/refresh token error: {} error_description:{}".format( + response.get("error", ""), response.get("error_description", ""))) helper.log_error("Please generate new code and update the input with new code.") postargs = { - 'severity': "error", - 'name': APP_NAME, - 'value': "Egnyte Add-on: Please generate new code and update the input with new code." + 'severity': "error", + 'name': APP_NAME, + 'value': "Egnyte Add-on: Please generate new code and update the input with new code." } rest.simpleRequest('/services/messages', - session_key, postargs=postargs) + session_key, postargs=postargs) return else: - state["access_token"] = response.get("access_token") - state["refresh_token"] = response.get("refresh_token") + # state["access_token"] = response.get("access_token") state["code"] = code set_checkpoint(helper, stanza_name, state) + + storage_passwords = service.storage_passwords + try: + # Retrieve existing password. This is safeguard in case of any racing condition. + # updating token is not necessary as it is deterministic based on client_id, secret & domain + body = storage_passwords.get(stanza_name + "/" + code)["body"] + except HTTPError: + storage_passwords.create(response.get("access_token"), stanza_name + "/" + code) + helper.log_debug("New storage password entry created.") except Exception as e: raise e checkpoint = get_checkpoint(helper, stanza_name) or dict() @@ -77,10 +94,13 @@ def collect_events(helper, ew): final_modifiedAfter = "" if checkpoint.get("modifiedAfter"): data_url = str(base_url) + "/api/v1/issueupdates?modifiedAfter=" + str(checkpoint.get("modifiedAfter")) - else: + else: data_url = str(base_url) + "/api/v1/issueupdates" data = {} modifiedAfter_done = True + + token = get_token_from_secure_password(stanza_name, code, service, helper, checkpoint, checkpoint) + while modifiedAfter_done: try: # collecting issues from the Egnyte server @@ -88,52 +108,13 @@ def collect_events(helper, ew): data_url = "{}&format=full".format(data_url) else: data_url = "{}?format=full".format(data_url) - data = collect_issues(helper, checkpoint.get('access_token'), data_url) + data = collect_issues(helper, token, data_url) except Exception as e: raise e # retrying to get new access token if token is expired if data == 401: - refresh_token = checkpoint.get('refresh_token') - try: - response = generate_or_refresh_token(helper=helper, auth_url=auth_url, clientid=clientid, client_secret=client_secret, - refresh_token=refresh_token) - if response.status_code == 401 or response.status_code == 400: - helper.log_error("Please generate new code and update the input with new code.") - postargs = { - 'severity': "error", - 'name': APP_NAME, - 'value': "Egnyte Add-on: Please generate new code and update the input with new code." - } - - rest.simpleRequest('/services/messages', - session_key, postargs=postargs) - return 0 - if response.status_code == 200: - response=response.json() - checkpoint["access_token"] = response.get("access_token") - checkpoint["refresh_token"] = response.get("refresh_token") - set_checkpoint(helper, stanza_name, checkpoint) - - except Exception as e: - raise e - - checkpoint = get_checkpoint(helper, stanza_name) - final_modifiedAfter = final_modifiedAfter or checkpoint.get("modifiedAfter") - if final_modifiedAfter: - data_url = str(base_url) + "/api/v1/issueupdates?modifiedAfter=" + str(final_modifiedAfter) - else: - data_url = str(base_url) + "/api/v1/issueupdates" - if format_value and "modifiedAfter" in data_url and "format" not in data_url: - data_url = "{}&format=full".format(data_url) - else: - data_url = "{}?format=full".format(data_url) - try: - data = collect_issues(helper, checkpoint.get('access_token'), data_url) - if data.get("error",""): - helper.log_error("Error while collecting data error: {} error_description:{}".format(response.get("error", ""), response.get("error_description", ""))) - return - except Exception as e: - raise e + helper.log_error("Please generate new code and update the input with new code.") + sys.exit(1) # indexing issues into Splunk if data.get("issues", ""): issues = data.get("issues") @@ -143,7 +124,8 @@ def collect_events(helper, ew): source = "egnyte" sourcetype = "egnyte:protect:incidents" for i in issues: - event = helper.new_event(data=json.dumps(i), time=event_time, host=None, index=index,source=source, sourcetype=sourcetype, done=True,unbroken=True) + event = helper.new_event(data=json.dumps(i), time=event_time, host=None, index=index, source=source, + sourcetype=sourcetype, done=True, unbroken=True) ew.write_event(event) number_of_events = number_of_events + event_count if data.get("modifiedAfter"): diff --git a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/_dummy_thread/__init__.py b/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/_dummy_thread/__init__.py deleted file mode 100644 index 63dced6..0000000 --- a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/_dummy_thread/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import absolute_import -import sys -__future_module__ = True - -if sys.version_info[0] < 3: - from dummy_thread import * -else: - raise ImportError('This package should not be accessible on Python 3. ' - 'Either you are trying to run from the python-future src folder ' - 'or your installation of python-future is corrupted.') diff --git a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/_markupbase/__init__.py b/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/_markupbase/__init__.py deleted file mode 100644 index 2909065..0000000 --- a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/_markupbase/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import absolute_import -import sys -__future_module__ = True - -if sys.version_info[0] < 3: - from markupbase import * -else: - raise ImportError('This package should not be accessible on Python 3. ' - 'Either you are trying to run from the python-future src folder ' - 'or your installation of python-future is corrupted.') diff --git a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/_thread/__init__.py b/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/_thread/__init__.py deleted file mode 100644 index 9f2a51c..0000000 --- a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/_thread/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import absolute_import -import sys -__future_module__ = True - -if sys.version_info[0] < 3: - from thread import * -else: - raise ImportError('This package should not be accessible on Python 3. ' - 'Either you are trying to run from the python-future src folder ' - 'or your installation of python-future is corrupted.') diff --git a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/backports/__init__.py b/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/backports/__init__.py deleted file mode 100644 index febdb2f..0000000 --- a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/backports/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# A Python "namespace package" http://www.python.org/dev/peps/pep-0382/ -# This always goes inside of a namespace package's __init__.py - -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/backports/configparser/__init__.py b/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/backports/configparser/__init__.py deleted file mode 100644 index a723399..0000000 --- a/src/TA-egnyte-protect/bin/ta_egnyte_protect/aob_py2/backports/configparser/__init__.py +++ /dev/null @@ -1,1401 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# flake8: noqa - -"""Configuration file parser. - -A configuration file consists of sections, lead by a "[section]" header, -and followed by "name: value" entries, with continuations and such in -the style of RFC 822. - -Intrinsic defaults can be specified by passing them into the -ConfigParser constructor as a dictionary. - -class: - -ConfigParser -- responsible for parsing a list of - configuration files, and managing the parsed database. - - methods: - - __init__(defaults=None, dict_type=_default_dict, allow_no_value=False, - delimiters=('=', ':'), comment_prefixes=('#', ';'), - inline_comment_prefixes=None, strict=True, - empty_lines_in_values=True, default_section='DEFAULT', - interpolation=, converters=): - Create the parser. When `defaults' is given, it is initialized into the - dictionary or intrinsic defaults. The keys must be strings, the values - must be appropriate for %()s string interpolation. - - When `dict_type' is given, it will be used to create the dictionary - objects for the list of sections, for the options within a section, and - for the default values. - - When `delimiters' is given, it will be used as the set of substrings - that divide keys from values. - - When `comment_prefixes' is given, it will be used as the set of - substrings that prefix comments in empty lines. Comments can be - indented. - - When `inline_comment_prefixes' is given, it will be used as the set of - substrings that prefix comments in non-empty lines. - - When `strict` is True, the parser won't allow for any section or option - duplicates while reading from a single source (file, string or - dictionary). Default is True. - - When `empty_lines_in_values' is False (default: True), each empty line - marks the end of an option. Otherwise, internal empty lines of - a multiline option are kept as part of the value. - - When `allow_no_value' is True (default: False), options without - values are accepted; the value presented for these is None. - - sections() - Return all the configuration section names, sans DEFAULT. - - has_section(section) - Return whether the given section exists. - - has_option(section, option) - Return whether the given option exists in the given section. - - options(section) - Return list of configuration options for the named section. - - read(filenames, encoding=None) - Read and parse the iterable of named configuration files, given by - name. A single filename is also allowed. Non-existing files - are ignored. Return list of successfully read files. - - read_file(f, filename=None) - Read and parse one configuration file, given as a file object. - The filename defaults to f.name; it is only used in error - messages (if f has no `name' attribute, the string `' is used). - - read_string(string) - Read configuration from a given string. - - read_dict(dictionary) - Read configuration from a dictionary. Keys are section names, - values are dictionaries with keys and values that should be present - in the section. If the used dictionary type preserves order, sections - and their keys will be added in order. Values are automatically - converted to strings. - - get(section, option, raw=False, vars=None, fallback=_UNSET) - Return a string value for the named option. All % interpolations are - expanded in the return values, based on the defaults passed into the - constructor and the DEFAULT section. Additional substitutions may be - provided using the `vars' argument, which must be a dictionary whose - contents override any pre-existing defaults. If `option' is a key in - `vars', the value from `vars' is used. - - getint(section, options, raw=False, vars=None, fallback=_UNSET) - Like get(), but convert value to an integer. - - getfloat(section, options, raw=False, vars=None, fallback=_UNSET) - Like get(), but convert value to a float. - - getboolean(section, options, raw=False, vars=None, fallback=_UNSET) - Like get(), but convert value to a boolean (currently case - insensitively defined as 0, false, no, off for False, and 1, true, - yes, on for True). Returns False or True. - - items(section=_UNSET, raw=False, vars=None) - If section is given, return a list of tuples with (name, value) for - each option in the section. Otherwise, return a list of tuples with - (section_name, section_proxy) for each section, including DEFAULTSECT. - - remove_section(section) - Remove the given file section and all its options. - - remove_option(section, option) - Remove the given option from the given section. - - set(section, option, value) - Set the given option. - - write(fp, space_around_delimiters=True) - Write the configuration state in .ini format. If - `space_around_delimiters' is True (the default), delimiters - between keys and values are surrounded by spaces. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import functools -import io -import itertools -import os -import re -import sys -import warnings - -from backports.configparser.helpers import OrderedDict as _default_dict -from backports.configparser.helpers import ChainMap as _ChainMap -from backports.configparser.helpers import from_none, open, str, PY2 -from backports.configparser.helpers import PathLike, fspath -from backports.configparser.helpers import MutableMapping - -__all__ = ["NoSectionError", "DuplicateOptionError", "DuplicateSectionError", - "NoOptionError", "InterpolationError", "InterpolationDepthError", - "InterpolationMissingOptionError", "InterpolationSyntaxError", - "ParsingError", "MissingSectionHeaderError", - "ConfigParser", "SafeConfigParser", "RawConfigParser", - "Interpolation", "BasicInterpolation", "ExtendedInterpolation", - "LegacyInterpolation", "SectionProxy", "ConverterMapping", - "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"] - -DEFAULTSECT = "DEFAULT" - -MAX_INTERPOLATION_DEPTH = 10 - - -# exception classes -class Error(Exception): - """Base class for ConfigParser exceptions.""" - - def __init__(self, msg=''): - self.message = msg - Exception.__init__(self, msg) - - def __repr__(self): - return self.message - - __str__ = __repr__ - - -class NoSectionError(Error): - """Raised when no section matches a requested option.""" - - def __init__(self, section): - Error.__init__(self, 'No section: %r' % (section,)) - self.section = section - self.args = (section, ) - - -class DuplicateSectionError(Error): - """Raised when a section is repeated in an input source. - - Possible repetitions that raise this exception are: multiple creation - using the API or in strict parsers when a section is found more than once - in a single input file, string or dictionary. - """ - - def __init__(self, section, source=None, lineno=None): - msg = [repr(section), " already exists"] - if source is not None: - message = ["While reading from ", repr(source)] - if lineno is not None: - message.append(" [line {0:2d}]".format(lineno)) - message.append(": section ") - message.extend(msg) - msg = message - else: - msg.insert(0, "Section ") - Error.__init__(self, "".join(msg)) - self.section = section - self.source = source - self.lineno = lineno - self.args = (section, source, lineno) - - -class DuplicateOptionError(Error): - """Raised by strict parsers when an option is repeated in an input source. - - Current implementation raises this exception only when an option is found - more than once in a single file, string or dictionary. - """ - - def __init__(self, section, option, source=None, lineno=None): - msg = [repr(option), " in section ", repr(section), - " already exists"] - if source is not None: - message = ["While reading from ", repr(source)] - if lineno is not None: - message.append(" [line {0:2d}]".format(lineno)) - message.append(": option ") - message.extend(msg) - msg = message - else: - msg.insert(0, "Option ") - Error.__init__(self, "".join(msg)) - self.section = section - self.option = option - self.source = source - self.lineno = lineno - self.args = (section, option, source, lineno) - - -class NoOptionError(Error): - """A requested option was not found.""" - - def __init__(self, option, section): - Error.__init__(self, "No option %r in section: %r" % - (option, section)) - self.option = option - self.section = section - self.args = (option, section) - - -class InterpolationError(Error): - """Base class for interpolation-related exceptions.""" - - def __init__(self, option, section, msg): - Error.__init__(self, msg) - self.option = option - self.section = section - self.args = (option, section, msg) - - -class InterpolationMissingOptionError(InterpolationError): - """A string substitution required a setting which was not available.""" - - def __init__(self, option, section, rawval, reference): - msg = ("Bad value substitution: option {0!r} in section {1!r} contains " - "an interpolation key {2!r} which is not a valid option name. " - "Raw value: {3!r}".format(option, section, reference, rawval)) - InterpolationError.__init__(self, option, section, msg) - self.reference = reference - self.args = (option, section, rawval, reference) - - -class InterpolationSyntaxError(InterpolationError): - """Raised when the source text contains invalid syntax. - - Current implementation raises this exception when the source text into - which substitutions are made does not conform to the required syntax. - """ - - -class InterpolationDepthError(InterpolationError): - """Raised when substitutions are nested too deeply.""" - - def __init__(self, option, section, rawval): - msg = ("Recursion limit exceeded in value substitution: option {0!r} " - "in section {1!r} contains an interpolation key which " - "cannot be substituted in {2} steps. Raw value: {3!r}" - "".format(option, section, MAX_INTERPOLATION_DEPTH, - rawval)) - InterpolationError.__init__(self, option, section, msg) - self.args = (option, section, rawval) - - -class ParsingError(Error): - """Raised when a configuration file does not follow legal syntax.""" - - def __init__(self, source=None, filename=None): - # Exactly one of `source'/`filename' arguments has to be given. - # `filename' kept for compatibility. - if filename and source: - raise ValueError("Cannot specify both `filename' and `source'. " - "Use `source'.") - elif not filename and not source: - raise ValueError("Required argument `source' not given.") - elif filename: - source = filename - Error.__init__(self, 'Source contains parsing errors: %r' % source) - self.source = source - self.errors = [] - self.args = (source, ) - - @property - def filename(self): - """Deprecated, use `source'.""" - warnings.warn( - "The 'filename' attribute will be removed in future versions. " - "Use 'source' instead.", - DeprecationWarning, stacklevel=2 - ) - return self.source - - @filename.setter - def filename(self, value): - """Deprecated, user `source'.""" - warnings.warn( - "The 'filename' attribute will be removed in future versions. " - "Use 'source' instead.", - DeprecationWarning, stacklevel=2 - ) - self.source = value - - def append(self, lineno, line): - self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, line) - - -class MissingSectionHeaderError(ParsingError): - """Raised when a key-value pair is found before any section header.""" - - def __init__(self, filename, lineno, line): - Error.__init__( - self, - 'File contains no section headers.\nfile: %r, line: %d\n%r' % - (filename, lineno, line)) - self.source = filename - self.lineno = lineno - self.line = line - self.args = (filename, lineno, line) - - -# Used in parser getters to indicate the default behaviour when a specific -# option is not found it to raise an exception. Created to enable `None' as -# a valid fallback value. -_UNSET = object() - - -class Interpolation(object): - """Dummy interpolation that passes the value through with no changes.""" - - def before_get(self, parser, section, option, value, defaults): - return value - - def before_set(self, parser, section, option, value): - return value - - def before_read(self, parser, section, option, value): - return value - - def before_write(self, parser, section, option, value): - return value - - -class BasicInterpolation(Interpolation): - """Interpolation as implemented in the classic ConfigParser. - - The option values can contain format strings which refer to other values in - the same section, or values in the special default section. - - For example: - - something: %(dir)s/whatever - - would resolve the "%(dir)s" to the value of dir. All reference - expansions are done late, on demand. If a user needs to use a bare % in - a configuration file, she can escape it by writing %%. Other % usage - is considered a user error and raises `InterpolationSyntaxError'.""" - - _KEYCRE = re.compile(r"%\(([^)]+)\)s") - - def before_get(self, parser, section, option, value, defaults): - L = [] - self._interpolate_some(parser, option, L, value, section, defaults, 1) - return ''.join(L) - - def before_set(self, parser, section, option, value): - tmp_value = value.replace('%%', '') # escaped percent signs - tmp_value = self._KEYCRE.sub('', tmp_value) # valid syntax - if '%' in tmp_value: - raise ValueError("invalid interpolation syntax in %r at " - "position %d" % (value, tmp_value.find('%'))) - return value - - def _interpolate_some(self, parser, option, accum, rest, section, map, - depth): - rawval = parser.get(section, option, raw=True, fallback=rest) - if depth > MAX_INTERPOLATION_DEPTH: - raise InterpolationDepthError(option, section, rawval) - while rest: - p = rest.find("%") - if p < 0: - accum.append(rest) - return - if p > 0: - accum.append(rest[:p]) - rest = rest[p:] - # p is no longer used - c = rest[1:2] - if c == "%": - accum.append("%") - rest = rest[2:] - elif c == "(": - m = self._KEYCRE.match(rest) - if m is None: - raise InterpolationSyntaxError(option, section, - "bad interpolation variable reference %r" % rest) - var = parser.optionxform(m.group(1)) - rest = rest[m.end():] - try: - v = map[var] - except KeyError: - raise from_none(InterpolationMissingOptionError( - option, section, rawval, var)) - if "%" in v: - self._interpolate_some(parser, option, accum, v, - section, map, depth + 1) - else: - accum.append(v) - else: - raise InterpolationSyntaxError( - option, section, - "'%%' must be followed by '%%' or '(', " - "found: %r" % (rest,)) - - -class ExtendedInterpolation(Interpolation): - """Advanced variant of interpolation, supports the syntax used by - `zc.buildout'. Enables interpolation between sections.""" - - _KEYCRE = re.compile(r"\$\{([^}]+)\}") - - def before_get(self, parser, section, option, value, defaults): - L = [] - self._interpolate_some(parser, option, L, value, section, defaults, 1) - return ''.join(L) - - def before_set(self, parser, section, option, value): - tmp_value = value.replace('$$', '') # escaped dollar signs - tmp_value = self._KEYCRE.sub('', tmp_value) # valid syntax - if '$' in tmp_value: - raise ValueError("invalid interpolation syntax in %r at " - "position %d" % (value, tmp_value.find('$'))) - return value - - def _interpolate_some(self, parser, option, accum, rest, section, map, - depth): - rawval = parser.get(section, option, raw=True, fallback=rest) - if depth > MAX_INTERPOLATION_DEPTH: - raise InterpolationDepthError(option, section, rawval) - while rest: - p = rest.find("$") - if p < 0: - accum.append(rest) - return - if p > 0: - accum.append(rest[:p]) - rest = rest[p:] - # p is no longer used - c = rest[1:2] - if c == "$": - accum.append("$") - rest = rest[2:] - elif c == "{": - m = self._KEYCRE.match(rest) - if m is None: - raise InterpolationSyntaxError(option, section, - "bad interpolation variable reference %r" % rest) - path = m.group(1).split(':') - rest = rest[m.end():] - sect = section - opt = option - try: - if len(path) == 1: - opt = parser.optionxform(path[0]) - v = map[opt] - elif len(path) == 2: - sect = path[0] - opt = parser.optionxform(path[1]) - v = parser.get(sect, opt, raw=True) - else: - raise InterpolationSyntaxError( - option, section, - "More than one ':' found: %r" % (rest,)) - except (KeyError, NoSectionError, NoOptionError): - raise from_none(InterpolationMissingOptionError( - option, section, rawval, ":".join(path))) - if "$" in v: - self._interpolate_some(parser, opt, accum, v, sect, - dict(parser.items(sect, raw=True)), - depth + 1) - else: - accum.append(v) - else: - raise InterpolationSyntaxError( - option, section, - "'$' must be followed by '$' or '{', " - "found: %r" % (rest,)) - - -class LegacyInterpolation(Interpolation): - """Deprecated interpolation used in old versions of ConfigParser. - Use BasicInterpolation or ExtendedInterpolation instead.""" - - _KEYCRE = re.compile(r"%\(([^)]*)\)s|.") - - def before_get(self, parser, section, option, value, vars): - rawval = value - depth = MAX_INTERPOLATION_DEPTH - while depth: # Loop through this until it's done - depth -= 1 - if value and "%(" in value: - replace = functools.partial(self._interpolation_replace, - parser=parser) - value = self._KEYCRE.sub(replace, value) - try: - value = value % vars - except KeyError as e: - raise from_none(InterpolationMissingOptionError( - option, section, rawval, e.args[0])) - else: - break - if value and "%(" in value: - raise InterpolationDepthError(option, section, rawval) - return value - - def before_set(self, parser, section, option, value): - return value - - @staticmethod - def _interpolation_replace(match, parser): - s = match.group(1) - if s is None: - return match.group() - else: - return "%%(%s)s" % parser.optionxform(s) - - -class RawConfigParser(MutableMapping): - """ConfigParser that does not do interpolation.""" - - # Regular expressions for parsing section headers and options - _SECT_TMPL = r""" - \[ # [ - (?P
[^]]+) # very permissive! - \] # ] - """ - _OPT_TMPL = r""" - (?P