Merge pull request #2 from egnyte/release/1.1.1

Release/1.1.1

src/TA-egnyte-protect/TA-egnyte-protect.aob_meta

@@ -0,0 +1 @@
+{"basic_builder": {"appname": "TA-egnyte-protect", "friendly_name": "Egnyte Secure and Govern Add-on For Splunk", "version": "1.1.1", "author": "", "description": "", "theme": "#65A637", "large_icon": "iVBORw0KGgoAAAANSUhEUgAAAEgAAABICAYAAABV7bNHAAAAAXNSR0IArs4c6QAACHhJREFUeF7t3Idu3LwSBWA5vffee/L+TxMgvffeq398uvcYtKLtq7UNiMDC2RVFcc6cOTOkiCxVfRuKwFKPz3AEeoBGMKQHqAdoNhHpGTQtg27evLk8G/Yb6+4bN260kmUgg3qA/ufgHqD/E71n0IiI7wHqAZotKfQM2sgM2rFjR7Vnz54VE5aXl6vPnz9XP378mI0WE9y9rhl06tSpCkg/f/6sTdq2bVv15cuX6sWLFxOYOFvXNQVo06ZN1f79+2tW/Pr1a5UlS0tL1blz56qPHz9W7969q68dOXKkBuzJkycVNpUNeLt27ao+fPjwz7VZIFozgPbt21cdPXq02rx5cx0yDx8+rLZv317t3bu3NtS/gfTo0aPq69evtY2uYdXfv3/re/z+6dOnGtzz589XW7durX7//l29fPmyBn0ebeEAAeT06dM1AG/evKk9zji/Y5QQYvj3799rEP78+bPKTv2wyP27d++uwQQYkIB58ODB6tChQ/UYT58+ra/N0hYOEC9funSpevz4cQ1GtIWh2NAEZJRxW7ZsqYXcWAlTTDt58mR1+/btjQcQgzEIEM+fPx9l/1TX5zn+whkkNM6cOVOHwLNnz/4BgO7s3Lmz/uib0CPKQJXRhJ/7B7HN+EQbS5MBp0K6qqqFAiSMeFdmIqRlJmIQ/RAeQAEC44QNHaFPftePBgFSWMlwEfGA4JoQ8zwgGWva1ilA8T4jsUHafvv2bS3OafrIZtI9Q96/f19r0TBxBQDj3QPQb9++1bVRs4A8duxYJVvKkMCmf1g3iXB3ChBAhIqUy+v+lkUeI3nahDEqoj2JtzEKEMZ69erVSs2UMcpiU9YzB3XUuK0zgHju+PHjdarlZV4nygkr3ncdW/K7PoyVlbDB7+qasunjPsaWfQ4cOFDfK3w5Ic/RH0jGAY4QB9C4zugEIHpx8eLF2ptCqtmAhzk8Xl6nQYcPH67ZxGBGAbhsahz9Xr9+XYdY2ScaF9DbWCKcOeD+/ftjVdydAMQAE7lz584/mUbInT17tgaGkWUDGs9jDuOBdffu3Yn6AEkWM3abc1KHyaDYNqp1ApBJoDUBLlMtZl24cKEW4yYzTLQJkLUXkMuWpcYwEBNuxLnMYHRQiEkawmycorQTgBgk9q9cuVKHkcykYZXwunfvXmsmmRdAKUZV2Q8ePFjBFyOBh5XNxe4gJnUGEG9ZYyn3eQqbLl++XIfPIGrPEyAspoOYmoWrjOc3ADV3DxYOELaofYSYxnt0BXvavIdxJ06cWKVBQqxNg5QFZYgNGtN4gLKITeM0gJW12DAd6oxBJiKbRCh5zvdSmE2eEYQ1TThK08l0bZOXHWW6Zh/FoHtTWUsIajE6lnKh6bg1EWlsuHr1as0ek03mkFrLahdoJs7gVLfZ4jBGlhRNIwb1UW9hKdYFkOigbRVNH3WU0B+nzZVBDFKnAMS/oz8mhSm3bt1amRMBpUlN0MaZ9LA+HINF0TlZC1ip4KNDWesJtWEL2rkCRDPQHiOy4xf9UZyVGUXKZ0y5YzgrOE3mJnNyVqlDqnBAcSYJyJZu2/PnDpDJRJjzQEsArGqugYQY6rcVdNOAFc0Jc42haJXaCXmz0clyz3vNABLz0nyzOJTWeb2taJwGIGD4lJkPOJjSzIbGXzhASeW8IsRkrUE0j3ebBk0DTO5pAxw4NLAMbzIgxIAnq6aQ7ZxB6h4PzSuYUN0kTYogl60tJGYBqC1khXd2F40dkbaaJ870Z1jROFcNinFZZggdEyHQ1lCyWFkk6nft2rVVG/jTAjRI9C2MsVndpOV1U3ONN+i5nQDkYQo0FSsBHpaxLF6F5KxCHTZyQmqqZLVyuaFQxKJxN806A0jKN+mkV4JoazSejMfURwCcVajbspV0rg4qt104ROYcltpLNnUGED0ymZT5Klx64Hu5Jzwsy0wSbgRaK9+U2BeyUM5vmdO6WKxihUo52x2+e2Goci2911z1TwJK2ZdAy0YZO2DYE8Jcbd1sd4hx1CbIYj1ZQhgIPUVb1kp0ApCj0u0w4AJGuWyhgdhTas2geQ0bu5MQy45e25YrLTLxstoGmnvyemYSFmGmTAXwgKGswBaANVN4Fs40L3XawgHCCpSXnZr7zryYCjYLSP0ZafL0ovkicJABxiLyqnTgAj7v5QdtzKnqhXVZOC4cIA9M7cNgtQcQcq4nG+s0I1kNE0xeX3WLD++XAGeBaazsGGCBMUpw6Fw2xIybFb2aDKAya3RpFFs7CbE8VBYBBib5W27WS8GKRxMFYjbQ6Qlwc1aorGvyWsh4wMvBK4AJKZ/yVZLfgWNrBeDYBdC2MwELLxQ9kPc0aZ33c2IsrIloChEex6hU2slu5W4gbcm2bQwCfBbDQja6ki1c14UTjfIccxl3w94zOmVQ0yuMLk92mChDwgyTV8T5RMfKmgVDsqYDFGHHNP0xJyzkGOx0TTjNcsJjoQBlsQgklC9TMO8ynuGaLNfccZTthAkBBoBQA04TAIKfd3LNV9ejNKd5feEAmQCAaMig06rYk3VUWegFoLbNr9Kwtqp6UmDSf+EApQ7BnryvEnpEm36UbLh+/fqqLdls6ZabXxH18l5MlK3a6rBJgVo4QPSBhwFiWSBEhIQm2+T0mL/6lStxAAlBepNDnP4KI/olLCPk5amRSUEp+y8coDwcQNnMkuqJKYB43y4AVtERGa88J02oARIgsdB3xSew/O6eceucUeCtGUAmRmeIrgKu7SBB8+VjDpI3XwoYqywcRxk9yfU1BWjURHPOMJkICDnfOOreeV1f1wApJIVimrqpPA89LxCGjbOuAVoEAKOe0QM0AqEeoB6gUUE0/HrPoJ5BPYNmQ2DeDOp0Nhto8P5/fxnhrB6gHqDZ4rlnUM+g2Rj0H1dT7aMuXxb/AAAAAElFTkSuQmCC", "small_icon": "iVBORw0KGgoAAAANSUhEUgAAACQAAAAkCAYAAADhAJiYAAAAAXNSR0IArs4c6QAAAxJJREFUWEftmIlOIlEQRQtxQVwQcEk0uIAC//81gIioEeK+ICBqYHIqU+QJCE0PcYjxJSa2TXfduksVMSBTdgJThkemG1Aul+v8D8ay2WyXmE8M/QL6K8fPZmhxcVE6nY7MzMzIx8eHvL29jW1D3wwFg0FJpVLSarWk0WhILBaTWq0mlUpFQezs7Mjy8rLc3d0JQPkpFovSbreHgvQNaG1tTZ6enpQRLwfWlpaWFPSw4xvQ4eGhnJycKKBAICBbW1sSiUS6tWDi+vpaQXMAdHBwoM9MDBAyraysyPr6utLP2dvbk3q9Lre3t311AAoIgL28vOh9ZL6/v5fHx8eB7I7FUDqdluPjY30xzGQyGcnn83p9dHSkRj49PdXr7e1tCYfDUiqVVCr8VK1WlSnY4/P2LreTsQDBxtnZWReAvXB+fl4BJhIJBcAxsMiIbDDLMQ/t7+9LuVzuY3UsQBsbG3Jzc6PdEmmLtRdAVEY+Y5BUIl3vGQuQde1KFY1GBUC8HIaen5+1RjweVzmJf7PZ1Pt4inQ+PDx8ktuXZMwRijNnNjc31agwhpm9RH93d1fOz88FHxYKBQUP26+vr59I8sxQKBTShzHlwsKCdg1AuvVyMDnNGCCewewk1BdDJIaJTPRnZ2d1QgMOoPx92HE/x/yykYHRewelZ4bwCRJdXl7qALSBZ50PA+Qyaf5DQsbA+/u7P4bcKCeTyb54DwPkpguT05QbDF+S8RATGhO7rHw1T9wiVhymbEJ/5T/PklGAZFxcXGgt6xpvMXl702KA8BwSE3uXKXfI+maIhJgh8RSmJLr4AWPTPeuDMzc3pykCDNPd0gVAGsDc/7w67CsEc4itTcLY8rZOXAbN+CQSzzF7jFm+I5GuQfNrLMlcatnarAG6BQiFkY3kcGAPr11dXSlzMAOTtjq+CoFvQINMabuOYqurq901wjVrA5C2WiYOiAL4gDlCEcDwu217ZMJHrBjA4Tn8N2rN+GZoUId4CmnwGvL1Dj0vK2aigLwUHPWZX0A/h6FRnXzH/en+/9B3MDCqxh9YRC5DRODzCQAAAABJRU5ErkJggg==", "visible": false, "tab_version": "4.1.3", "tab_build_no": "0", "build_no": 1}}

src/TA-egnyte-protect/app.manifest

@@ -5,7 +5,7 @@
     "id": {
       "group": null,
       "name": "TA-egnyte-protect",
-      "version": "1.1.0"
+      "version": "1.1.1"
     },
     "author": [
       {

src/TA-egnyte-protect/bin/input_module_egnyte.py

@@ -2,29 +2,31 @@
 import os
 import sys
 import time
-import datetime
 import json
 from requests.exceptions import HTTPError
-from solnlib.splunkenv import get_splunkd_uri
-from solnlib.credentials import (CredentialManager, CredentialNotExistException)
-import requests
 from ta_egnyte_protect_utility import *
 import ta_egnyte_constants as tec
 import splunk.rest as rest
+import splunklib.client as client
+
 APP_NAME = os.path.abspath(__file__).split(os.sep)[-3]
 
+
 def validate_input(helper, definition):
     interval = float(definition.parameters.get('interval', None))
     if interval < 600:
         helper.log_error("Interval must be at least 600 seconds.")
         raise Exception('Interval must be at least 600 seconds')
 
+
 def get_checkpoint(helper, stanza_name):
     return helper.get_check_point(stanza_name)
 
+
 def set_checkpoint(helper, stanza_name, state):
     return helper.save_check_point(stanza_name, state)
 
+
 def collect_events(helper, ew):
     # getting setup parameters
     input_name = helper.get_input_stanza_names()
@@ -42,98 +44,77 @@ def collect_events(helper, ew):
     if endpoint == "US":
         base_url = tec.us_url
     else:
-        base_url =tec.europe_url
+        base_url = tec.europe_url
     auth_url = str(base_url) + "/oauth2/token"
-    
+
     checkpoint = get_checkpoint(helper, stanza_name) or dict()
+
+    service = client.connect(host='localhost', port=8089, token=session_key)
+
     # Going to take access/refresh token if it is not available in the checkpoint
     if not checkpoint or str(checkpoint.get("code")) != str(code):
-        helper.log_info("Checkpoint is not available or code changed from setup page. Hence requesting new access token.")
+        helper.log_info(
+            "Checkpoint is not available or code changed from setup page. Hence requesting new access token.")
         state = get_checkpoint(helper, stanza_name) or dict()
         try:
-            response = generate_or_refresh_token(helper=helper, auth_url=auth_url, clientid=clientid, client_secret=client_secret, code=code)
-            helper.log_info("Checkpoint is not available or code changed from setup page. Hence requested new access token.")
+            response = generate_or_refresh_token(helper=helper, auth_url=auth_url, clientid=clientid,
+                                                 client_secret=client_secret, code=code)
+            helper.log_info(
+                "Checkpoint is not available or code changed from setup page. Hence requested new access token.")
             response = response.json()
             if response.get("error"):
-                helper.log_error("Error while getting access/refresh token error: {} error_description:{}".format(response.get("error", ""), response.get("error_description", "")))
+                helper.log_error("Error while getting access/refresh token error: {} error_description:{}".format(
+                    response.get("error", ""), response.get("error_description", "")))
                 helper.log_error("Please generate new code and update the input with new code.")
                 postargs = {
-                        'severity': "error",
-                        'name': APP_NAME,
-                        'value': "Egnyte Add-on: Please generate new code and update the input with new code."
+                    'severity': "error",
+                    'name': APP_NAME,
+                    'value': "Egnyte Add-on: Please generate new code and update the input with new code."
                 }
                 rest.simpleRequest('/services/messages',
-                                session_key, postargs=postargs)
+                                   session_key, postargs=postargs)
                 return
             else:
-                state["access_token"] = response.get("access_token")
-                state["refresh_token"] = response.get("refresh_token")
+                # state["access_token"] = response.get("access_token")
                 state["code"] = code
                 set_checkpoint(helper, stanza_name, state)
+
+                storage_passwords = service.storage_passwords
+                try:
+                    # Retrieve existing password. This is safeguard in case of any racing condition.
+                    # updating token is not necessary as it is deterministic based on client_id, secret & domain
+                    body = storage_passwords.get(stanza_name + "/" + code)["body"]
+                except HTTPError:
+                    storage_passwords.create(response.get("access_token"), stanza_name + "/" + code)
+                    helper.log_debug("New storage password entry created.")
         except Exception as e:
             raise e
     checkpoint = get_checkpoint(helper, stanza_name) or dict()
     data_url = ""
     final_modifiedAfter = ""
     if checkpoint.get("modifiedAfter"):
         data_url = str(base_url) + "/api/v1/issueupdates?modifiedAfter=" + str(checkpoint.get("modifiedAfter"))
-    else:  
+    else:
         data_url = str(base_url) + "/api/v1/issueupdates"
     data = {}
     modifiedAfter_done = True
+
+    token = get_token_from_secure_password(stanza_name, code, service, helper, checkpoint, checkpoint)
+
     while modifiedAfter_done:
         try:
             # collecting issues from the Egnyte server
             if format_value and "modifiedAfter" in data_url and "format" not in data_url:
                 data_url = "{}&format=full".format(data_url)
             else:
                 data_url = "{}?format=full".format(data_url)
-            data = collect_issues(helper, checkpoint.get('access_token'), data_url)
+            data = collect_issues(helper, token, data_url)
         except Exception as e:
             raise e
         # retrying to get new access token if token is expired
         if data == 401:
-            refresh_token = checkpoint.get('refresh_token')
-            try:
-                response = generate_or_refresh_token(helper=helper, auth_url=auth_url, clientid=clientid, client_secret=client_secret, 
-                           refresh_token=refresh_token)
-                if response.status_code == 401 or response.status_code == 400:
-                    helper.log_error("Please generate new code and update the input with new code.")
-                    postargs = {
-                        'severity': "error",
-                        'name': APP_NAME,
-                        'value': "Egnyte Add-on: Please generate new code and update the input with new code."
-                    }
-
-                    rest.simpleRequest('/services/messages',
-                                    session_key, postargs=postargs)
-                    return 0
-                if response.status_code == 200:
-                    response=response.json()
-                    checkpoint["access_token"] = response.get("access_token")
-                    checkpoint["refresh_token"] = response.get("refresh_token")
-                    set_checkpoint(helper, stanza_name, checkpoint)
-
-            except Exception as e:
-                raise e
-
-            checkpoint = get_checkpoint(helper, stanza_name)
-            final_modifiedAfter = final_modifiedAfter or checkpoint.get("modifiedAfter")
-            if final_modifiedAfter:
-                data_url = str(base_url) + "/api/v1/issueupdates?modifiedAfter=" + str(final_modifiedAfter)
-            else:  
-                data_url = str(base_url) + "/api/v1/issueupdates"
-            if format_value and "modifiedAfter" in data_url and "format" not in data_url:
-                data_url = "{}&format=full".format(data_url)
-            else:
-                data_url = "{}?format=full".format(data_url)
-            try:
-                data = collect_issues(helper, checkpoint.get('access_token'), data_url)
-                if data.get("error",""):
-                   helper.log_error("Error while collecting data error: {} error_description:{}".format(response.get("error", ""), response.get("error_description", "")))
-                   return
-            except Exception as e:
-                raise e
+            helper.log_error("Please generate new code and update the input with new code.")
+            sys.exit(1)
         # indexing issues into Splunk
         if data.get("issues", ""):
             issues = data.get("issues")
@@ -143,7 +124,8 @@ def collect_events(helper, ew):
             source = "egnyte"
             sourcetype = "egnyte:protect:incidents"
             for i in issues:
-                event = helper.new_event(data=json.dumps(i), time=event_time, host=None, index=index,source=source, sourcetype=sourcetype, done=True,unbroken=True)
+                event = helper.new_event(data=json.dumps(i), time=event_time, host=None, index=index, source=source,
+                                         sourcetype=sourcetype, done=True, unbroken=True)
                 ew.write_event(event)
             number_of_events = number_of_events + event_count
             if data.get("modifiedAfter"):