Splunk_TA_paloalto.aob_meta

{"basic_builder": {"appname": "Splunk_TA_paloalto", "friendly_name": "Palo Alto Networks Add-on for Splunk", "version": "7.0.0", "author": "Palo Alto Networks", "description": "", "theme": "#FFFFFF", "large_icon": "iVBORw0KGgoAAAANSUhEUgAAAEgAAABICAYAAABV7bNHAAAFiklEQVR4nO2bXWgcVRTHtxZKGhMVjY2CjbYo1eI5s5tpGkpe1hdFH3wL3DP74bbFLaRFaEVBkXTrJ7RWH1QQH8S2ULS+CULfBEGlWrDSb8GPtvjVFVJbAvXeweNDsmuy2d05s3t3Z4X5w31L/nPPL+ecOXNnkkjEihUrVqxYsWLFihWrR2U8LPmET9jyYy+ZNJ7zhi2/SKUJp7QC1gqYvdTdljy/1wpYE75nwy8yaXK2VeBoBWwIZ/wMPNqqH+fdEUPwzWJP2G9zz12TVrB9YSCLV2pDWD/OjN9kCMv1/fBgJ2LomGozp3YZwhn2kkmpH0+lBwzhsWaevue83cmYrKl55tQGlXwsyG+urPCSyJPwQDdibFlh4FQbdyF5TyM/PjK5vHFZNVo9Wm5a4VNh4TRr3Jx3RzTht614aoL3o2DQUK1kzhJQnvNQxY9pfDh85vRoJhlyptuFoxWwUXDdZJNpplHUCs/b8NQKP+TNqdujhPO0nUCq64wm+Mimp6/w8UjgaM/ZYTMQo+Bvzo7ey4V0nyY4a8WX8DCXSjd0HY4hfNkuHJxlGsWKP2+ZGAyaewI9yZnuOphEwk5DXgInN/pA7XWYeZkmON4anIgeP4zC3VbhEFytB6ciLrr9RuHnRsE/Uk+f4N1uMqnKOhyFs0zu/UHX5S0Tg0bBdRlwfE4US8Z52GpvMuTssps5WG6WORVxaXKFUfCF0PNNSSy+l8zMZ++nzLysbTia8APrmSM4F+Ki268VnrSaOYSv1pT4l23BsT3nGAXXRJlTSPdpBWdswvEVPFnXg+BQi3DwdZtwtIILnMM1gXCKbr8m+E4I/EVhLM8EQD7KOzetDAEH9lsvKykcy5lTW1ZNYH8mhGO3Ic9ljqCstkwMiuF4zkuSWHwPt4baa1C5acIDVjOHsMw5vDEQTtHt14TnosicOutE3RHA/pwD10RzzlR6wH5ZtXdzMYSf1NK23ZDPiHrO1k23aoIfZGWFz8vgNG/IISAdZRofThjCj+1mDs4yjQ8Hwpm7W4me2rtQVvUX4bmE72HeEP5hyfCUqCHncI34SMNzdojgKHjB8h/6iu9hvrLhVUbBb+2Z4k98ZHJ5IJyi228IfpVljrNLBMdy5sy9lqqZ9jmTWh/mqXmxIfzJHqwNhJPfeJuOaM4Jtxq82OQsrtMKLoakfUxy7ss5XGMUXhb5krNNBEfhK5Yzpxz4QpPJHTIEf4kNBccG8w1ZdreKKHOMguucd0ck105wFkcDexLhKc7iukCvTGp9iDPm7SI49jNnxmSSEyI41cByuKpxueFJUUMmd8goNFbLyn5DLrf8WQ5vHltdp9wuiOacvDsin3PgWSGcvXbhwNW2v1niDN5V+XjAKPiai25/4O/k8EGjcFa0Sc95RLIPn/Ad+5kTfOcViT1YqwkPcyHdF/izU+kBQ/C7ZJO+h0py/Y7MOdKGbFNapTZohT+KN6pwd7AnHrSfOcn7usFjkebmnPCDpiF4rZGn9YM7wnIkmTPfkEVzToON710Kx3ZDjgiO9pyN0obcdC04zdMEh6zD2Ty2uutwOJty7AYC+ztwHn45ksxJJCoPuHjFHiBn2tb3RvOZ8wtn3TsjgVOFRO6QDvmAWzcYD0sVT+Nh6X9bVvU0D+lEG6W1p9bTEOxpGRDB2cjKqpF456aVrWRS9dSujhq+DQ3KnKJ7czdjF4vJHar+H4UkGIX7gjx9BW+FgHOp5zKnVlyaXFH7/xR1lwdFqaf2oCgoq9M8lR7oZGzWxIXkLc3KTXrWvFDN3vgawjIXxu7oRCwd0/xZ0FeShixV3cZNcLzny6qZNOHP/zVkJ9eun+85uQWALtrYY6Saa9xw2uaHlUbhPq3wfM/MObFixYoVK1asWLFixYoVTv8CfUr/kdHM+XgAAAAASUVORK5CYII=", "small_icon": "iVBORw0KGgoAAAANSUhEUgAAACQAAAAkCAYAAADhAJiYAAADEUlEQVRYhe2WS2sTURTHLxVBdCWIWnwlLZSU3HOzEERaH1CtrUo3tck9d9JWV34CUagLuxBEpAguxI2iCxcuXNiVglJ8fAWL2mofCLrUapvMPYPXRTJt5pFkGJKFmD+c3X/m/O553BnGWmrpfxchn9BKnK/rU6KPFNxsNswljWC0BKoFRYr3a8lXNIIh5LeaC+OGBNJSjAd94gQh/+nxKphqKIydg8ueBBVhK55bh1GZIxpBh/lIwu3GwCCfqAazHpYYp1xmoBpMRVXvmIsHN8eGIYQrdWEQDCG81Io/iuD7vqK6dsSDkeJqNBjxzoyJbSbLNmmExzW8y8bqPhCzMvxaRJi3ZujgVvc5M8naCOFZiPdzUaa6KnMYC7ZHgrEVvx6xTTOVMIwxpvPiMCH88MMYyTs9MFmR1AizjoIHDamMRnhVBea3z7dQyKb3V/rWkO/TEj5sePhdM8naYsMQwoy5kNjieTaX7vXD2MgX/TAmm96tkS+GbN89L4yCG5FgFLz2n4Zy6V4tYdXfpgBMXuwNhdmIhy7MVCQYKZ4HKlO6CNcCMxOojEhqhPkIeZ4wB7nUCHadbXrqnxmy4GRIZebWspk9ARgJX+rCSCBHQp4xxpijxDlCXghvE58OVMbKDARnBj7621TIimSdNrmhHSnQM0eOEkP+ShHCC+YTKdHnb5ON/FNsGAnkoBjx5ykn4/0agcozMx0KI+GP76XzZrS73dMmCzqiwBACORacCYWpSDroINwPbJMSgxrB8Z6Ov/ffuCYrkoTwNVJl6sFUk4NihILbZGyEBZMVSddXlLyTEL5FmhnkZ+PByMxwyGpXxpJRkDIKUhphKRJMHk7HgjFjYqeWpZmqE2/KUc9XJCUGY8G4Km/VrxpJFgqYShQwldAICzVmZjV2ZYJQcJwwcAkaG2HZKL7L9Znh7nZbhgy0hCJJfqohMK5KX3PP5TnnX3fGGDOj3e0aYa5yZsiCYw2FWU9mZQ7pEtRsAVOJar5y+2YJeYGQ9zQFxhUh7yn6frjCZCzoIBRHmwrTUkv/gv4CNzePiwDo44MAAAAASUVORK5CYII=", "visible": true, "tab_version": "4.0.0", "tab_build_no": "0", "build_no": 4}, "data_input_builder": {"datainputs": [{"index": "default", "sourcetype": "aperture", "interval": "30", "use_external_validation": true, "streaming_mode_xml": true, "name": "aperture", "title": "Aperture", "description": "", "type": "customized", "parameters": [{"required": true, "name": "region", "label": "Region", "default_value": "us", "placeholder": "", "help_string": "", "possible_values": [{"value": "us", "label": "US"}, {"value": "eu", "label": "Europe"}, {"value": "apac", "label": "Asia Pacific"}], "type": "dropdownlist", "format_type": "dropdownlist", "value": "us"}, {"required": true, "name": "global_account", "label": "Global Account", "default_value": "", "placeholder": "", "help_string": "", "possible_values": [], "type": "global_account", "format_type": "global_account", "value": ""}], "data_inputs_options": [{"type": "customized_var", "name": "region", "title": "Region", "description": "", "required_on_edit": false, "required_on_create": true, "possible_values": [{"value": "us", "label": "US"}, {"value": "eu", "label": "Europe"}, {"value": "apac", "label": "Asia Pacific"}], "format_type": "dropdownlist", "default_value": "us", "placeholder": ""}, {"type": "customized_var", "name": "global_account", "title": "Global Account", "description": "", "required_on_edit": false, "required_on_create": true, "possible_values": [], "format_type": "global_account", "default_value": "", "placeholder": ""}], "code": "\n# encoding = utf-8\n\nimport os\nimport sys\nimport time\nimport datetime\nimport json\nimport base64\n\n'''\n    IMPORTANT\n    Edit only the validate_input and collect_events functions.\n    Do not edit any other part in this file.\n    This file is generated only once when creating the modular input.\n'''\n'''\n# For advanced users, if you want to create single\ninstance mod input, uncomment this method.\ndef use_single_instance_mode():\n    return True\n'''\n\nREGION_DOMAIN = {\n    'us': 'api.aperture.paloaltonetworks.com',\n    'eu': 'api.aperture-eu.paloaltonetworks.com',\n    'apac': 'api.aperture-apac.paloaltonetworks.com',\n}\n\n\ndef validate_input(helper, definition):\n    \"\"\"Implement your own validation logic to\n        validate the input stanza configurations\"\"\"\n    # This example accesses the modular input variable\n    opt_global_account = definition.parameters.get('global_account', None)\n    pass\n\n\ndef get_auth_token(helper, opt_global_account, proxy_enabled):\n    helper.log_debug(\"Start get_auth_token.\")\n    client_id = opt_global_account['username']\n    secret = opt_global_account['password']\n    region = helper.get_arg('region')\n    url_domain = REGION_DOMAIN[region]\n    url = \"https://{0}/oauth/token\".format(url_domain)\n    method = \"POST\"\n    parameters = {'scope': 'api_access',\n                  'grant_type': 'client_credentials'}\n    auth = base64.b64encode('{0}:{1}'.format(client_id, secret).encode('ascii'))\n    auth = auth.decode('utf-8')\n    header = {'Authorization': 'Basic ' + auth,\n              'Content-Type': 'application/x-www- \\\n              form-urlencoded; charset=ISO-8859-1',\n              'Accept': 'application/json'}\n    response = helper.send_http_request(url, method, parameters=parameters,\n                                        payload=None, headers=header,\n                                        cookies=None, verify=True, cert=None,\n                                        timeout=30, use_proxy=proxy_enabled)\n    r_status = response.status_code\n    if r_status == 200:\n        helper.log_debug('Token recieved')\n        token = response.json()['access_token']\n        return token\n    elif r_status == 401:\n        helper.log_error('ERROR: Invalid credentials.')\n        raise ValueError(r_status)\n    else:\n        helper.log_error('ERROR: Unable to retrieve token.')\n        helper.log_debug(r_status)\n        raise ValueError(r_status)\n\n\ndef collect_events(helper, ew):\n    log_level = helper.get_log_level()\n    helper.set_log_level(log_level)\n    opt_global_account = helper.get_arg('global_account')\n    region = helper.get_arg('region')\n    url_domain = REGION_DOMAIN[region]\n    proxy_settings = helper.get_proxy()\n    proxy_enabled = bool(proxy_settings)\n    helper.log_debug(\"Checking if Proxy is enabled\")\n    helper.log_debug(proxy_enabled)\n    helper.log_debug(\"Current input type is set to:\")\n    helper.log_debug(helper.get_input_stanza_names())\n    token = get_auth_token(helper, opt_global_account, proxy_enabled)\n    headers = {'Authorization': 'Bearer ' + token}\n    method = 'GET'\n    url = \"https://{0}/api/v1/log_events_bulk\".format(url_domain)\n    r_status = 200\n    while r_status != 204:\n        response = helper.send_http_request(\n            url, method, parameters=None, payload=None,\n            headers=headers, cookies=None, verify=True, cert=None,\n            timeout=30, use_proxy=proxy_enabled)\n        r_status = response.status_code\n        helper.log_debug('API status code is:')\n        helper.log_debug(r_status)\n        if r_status == 200:\n            helper.log_debug(\"Adding data to index.\")\n            events = response.json()['events']\n            for data in events:\n                helper.log_debug(data)\n                timestamp = datetime.datetime.strptime(data['timestamp'], '%Y-%m-%dT%H:%M:%SZ')\n                final_time = (timestamp - datetime.datetime.fromtimestamp(0)).total_seconds()\n                helper.log_debug(final_time)\n                try:\n                    event = helper.new_event(\n                        host=url_domain,\n                        source=helper.get_input_stanza_names(),\n                        index=helper.get_output_index(),\n                        sourcetype=helper.get_sourcetype(),\n                        time=final_time,\n                        data=json.dumps(data))\n                    ew.write_event(event)\n                except Exception as e:\n                    ew.log_error('Error on parse event. ' + str(e))\n        elif r_status == 204:\n            helper.log_debug(\"STATUS 204: No new events were found.\")\n            break\n        elif r_status >= 400:\n            helper.log_debug(\"ERROR Status is:\")\n            helper.log_debug(r_status)\n            raise ValueError(r_status)\n        else:\n            helper.log_error('There was a problem when trying to collect events using the aperture API call.')\n", "customized_options": [{"name": "region", "value": "us"}, {"name": "global_account", "value": ""}], "uuid": "0e312910c3d249f78b8e2386a4ddeaef"}, {"index": "default", "sourcetype": "AutoFocus", "interval": "60", "use_external_validation": true, "streaming_mode_xml": true, "name": "autofocus_export", "title": "AutoFocus Export", "description": "", "type": "customized", "parameters": [{"name": "label", "label": "Label", "help_string": "", "required": false, "format_type": "text", "default_value": "", "placeholder": "", "type": "text", "value": ""}], "data_inputs_options": [{"type": "customized_var", "name": "label", "title": "Label", "description": "", "required_on_edit": false, "required_on_create": false, "format_type": "text", "default_value": "", "placeholder": ""}], "code": "\n# encoding = utf-8\n\nimport os\nimport sys\nimport time\nimport datetime\n\nlibpath = os.path.dirname(os.path.abspath(__file__))\nsys.path[:0] = [os.path.join(libpath, 'lib')]\nimport common\nimport pan.afapi\nimport json\n\nfrom kvstore import KvStoreHandler\n\n'''\n    IMPORTANT\n    Edit only the validate_input and collect_events functions.\n    Do not edit any other part in this file.\n    This file is generated only once when creating the modular input.\n'''\n# For advanced users, if you want to create single instance mod input, uncomment this method.\ndef use_single_instance_mode():\n    return True\n\n\ndef validate_input(helper, definition):\n    \"\"\"Implement your own validation logic to validate the input stanza configurations\"\"\"\n    # This example accesses the modular input variable\n    opt_label = definition.parameters.get('label', None)\n    pass\n\n\ndef collect_events(helper, ew):\n    # Implement your data collection logic here\n\n    # The following examples get the arguments of this input.\n    # Note, for single instance mod input, args will be returned as a dict.\n    # For multi instance mod input, args will be returned as a single value.\n    opt_label = helper.get_arg('label')\n\n    # In single instance mode, to get arguments of a particular input, use\n    # opt_label = helper.get_arg('label', stanza_name)\n\n    # get input type\n    # helper.get_input_type()\n\n    # The following examples get input stanzas.\n    # get all detailed input stanzas\n    # helper.get_input_stanza()\n    # get specific input stanza with stanza name\n    # helper.get_input_stanza(stanza_name)\n    # get all stanza names\n    # helper.get_input_stanza_names()\n\n    # The following examples get options from setup page configuration.\n    # get the loglevel from the setup page\n    loglevel = helper.get_log_level()\n    # get proxy setting configuration\n    # proxy_settings = helper.get_proxy()\n    # get global variable configuration\n    global_autofocus_api_key = helper.get_global_setting(\"autofocus_api_key\")\n    sessionKey = helper.context_meta['session_key']\n\n    # The following examples show usage of logging related helper functions.\n    # write to the log for this modular input using configured global log level or INFO as default\n    # helper.log(\"log message\")\n    # write to the log using specified log level\n    # helper.log_debug(\"log message\")\n    # helper.log_info(\"log message\")\n    # helper.log_warning(\"log message\")\n    # helper.log_error(\"log message\")\n    # helper.log_critical(\"log message\")\n    # set the log level for this modular input\n    # (log_level can be \"debug\", \"info\", \"warning\", \"error\" or \"critical\", case insensitive)\n    helper.set_log_level(loglevel)\n\n\n    # sessionKey = inputs.metadata.get('session_key')\n    for label in opt_label:\n        helper.log_debug(\"Current Label: \" + label)\n        # Check if Label already exsist and get last submit date\n        helper.log_debug(\"Getting AutoFocus Export for results\")\n        # Use API to get entries in Export List from AutoFocus\n        values = {\n            \"apiKey\": global_autofocus_api_key,\n            # \"panosFormatted\": \"true\",\n            \"exportMetadata\": \"true\",\n            \"label\": label\n        }\n        try:\n            afapi = pan.afapi.PanAFapi(api_key=global_autofocus_api_key)\n            jsAfapi = afapi.export(json.dumps(values)).json\n            af_export = jsAfapi['export_list']\n            # helper.log_debug(jsAfapi)\n        except pan.afapi.PanAFapiError as e:\n            helper.log_debug(e)\n            sys.exit(1)\n\n        sync_kvstore = sync_to_kvstore(helper, sessionKey, label, af_export)\n        helper.log_debug(sync_kvstore)\n        # Label does not exsist in KVstore go ahead and batch import.\n        if sync_kvstore == 1:\n            helper.log_debug(\"New to KVSTORE\")\n            send_to_kvstore(helper, sessionKey, jsAfapi['export_list'])\n        # Label does exsist in KVstore. Change Detected.\n        elif sync_kvstore == -1:\n            helper.log_debug(\"Update KVSTORE\")\n            # Delete entries for given label\n            options = {\n                \"app\": \"Splunk_TA_paloalto\",\n                \"owner\": \"nobody\",\n                \"collection\": \"autofocus_export\"\n            }\n            query = {\"label\": label}\n            delete = True\n            helper.log_debug(\"Delete entries for this label.\")\n            remove = KvStoreHandler.query(query, sessionKey, options, delete)\n            helper.log_debug(\"Add entries with this label to kvstore\")\n            send_to_kvstore(helper, sessionKey, jsAfapi['export_list'])\n        # NO CHANGE TO EXPORT LIST\n        else:\n            helper.log_debug(\"No Change\")\n\n    \"\"\"\n    # The following examples send rest requests to some endpoint.\n    response = helper.send_http_request(url, method, parameters=None, payload=None,\n                                        headers=None, cookies=None, verify=True, cert=None,\n                                        timeout=None, use_proxy=True)\n    # get the response headers\n    r_headers = response.headers\n    # get the response body as text\n    r_text = response.text\n    # get response body as json. If the body text is not a json string, raise a ValueError\n    r_json = response.json()\n    # get response cookies\n    r_cookies = response.cookies\n    # get redirect history\n    historical_responses = response.history\n    # get response status code\n    r_status = response.status_code\n    # check the response status, if the status is not sucessful, raise requests.HTTPError\n    response.raise_for_status()\n# The following examples show usage of check pointing related helper functions.\n    # save checkpoint\n    helper.save_check_point(key, state)\n    # delete checkpoint\n    helper.delete_check_point(key)\n    # get checkpoint\n    state = helper.get_check_point(key)\n\n    # To create a splunk event\n    helper.new_event(data, time=None, host=None, index=None, source=None, sourcetype=None, done=True, unbroken=True)\n    \"\"\"\n\n    '''\n    # The following example writes a random number as an event. (Multi Instance Mode)\n    # Use this code template by default.\n    import random\n    data = str(random.randint(0,100))\n    event = helper.new_event(source=helper.get_input_type(), index=helper.get_output_index(), sourcetype=helper.get_sourcetype(), data=data)\n    ew.write_event(event)\n    '''\n\n    '''\n    # The following example writes a random number as an event for each input config. (Single Instance Mode)\n    # For advanced users, if you want to create single instance mod input, please use this code template.\n    # Also, you need to uncomment use_single_instance_mode() above.\n    import random\n    input_type = helper.get_input_type()\n    for stanza_name in helper.get_input_stanza_names():\n        data = str(random.randint(0,100))\n        event = helper.new_event(source=input_type, index=helper.get_output_index(stanza_name), sourcetype=helper.get_sourcetype(stanza_name), data=data)\n        ew.write_event(event)\n    '''\n\n\ndef sync_to_kvstore(helper, sessionKey, label, af_export):\n    helper.log_debug(\"checking KVSTORE\")\n    url_options = {\n        \"app\": \"Splunk_TA_paloalto\",\n        \"owner\": \"nobody\",\n        \"collection\": \"autofocus_export\"\n    }\n    query = {\"label\": label}\n    arg = {\n        \"query\": query\n    }\n    response = KvStoreHandler.adv_query(arg, url_options, sessionKey)\n    # helper.log_debug(response)\n    results = 0\n    kv_export = json.loads(response[1])\n    # helper.log_debug(\"kv_export:\")\n    # helper.log_debug(kv_export)\n    # helper.log_debug(\"af_export:\")\n    # helper.log_debug(af_export)\n\n    # Check to see if we have entries in the KVstore already.\n    if kv_export:\n        helper.log_debug(\"Label Exist\")\n        # Check if list are same size\n        if len(kv_export) == len(af_export):\n            for entry in kv_export:\n                # Remove fields from kv_export so dicts will match.\n                if '_key' in entry:\n                    del(entry['_key'])\n                if '_user' in entry:\n                    del(entry['_user'])\n                if entry not in af_export:\n                    helper.log_debug(\"not a match\")\n                    helper.log_debug(entry)\n                    results = -1\n                    return results\n                else:\n                    helper.log_debug(\"Match\")\n        else:\n            helper.log_debug(\"List count not same.\")\n            results = -1\n            return results\n    else:\n        helper.log_debug(\"Label return empty\")\n        results = 1\n    return results\n\n\ndef send_to_kvstore(helper, sessionKey, export_list):\n    helper.log_debug(\"Inside Send to KVSTORE\")\n    url_options = {\n        \"app\": \"Splunk_TA_paloalto\",\n        \"owner\": \"nobody\",\n        \"collection\": \"autofocus_export\"\n    }\n    helper.log_debug(export_list)\n    response = KvStoreHandler.batch_create(export_list, sessionKey, url_options)\n    helper.log_debug(response)", "customized_options": [{"name": "label", "value": ""}], "uuid": "d79f8dd69d41446cb0817e00cc4c34d7", "sample_count": 0}, {"index": "default", "sourcetype": "pan:iot", "interval": "30", "use_external_validation": true, "streaming_mode_xml": true, "name": "iot_security", "title": "IoT Security", "description": "", "type": "customized", "parameters": [{"name": "customer_id", "label": "Customer ID", "help_string": "", "required": true, "format_type": "text", "default_value": "", "placeholder": "", "type": "text", "value": "banff-enterprise-demo"}, {"name": "access_key_id", "label": "Access Key ID", "help_string": "", "required": true, "format_type": "password", "default_value": "", "placeholder": "", "type": "password", "value": "1921124944:55d41f13516184710c76efc9cf8f40fb6d1d2a81293aa9c80a563849d53916fb"}, {"name": "secret_access_key", "label": "Secret Access Key", "help_string": "", "required": true, "format_type": "password", "default_value": "", "placeholder": "", "type": "password", "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiI1ZjE3NmZmZmQ5ZTVmZjFhNzAwOGVmYmQiLCJlbWFpbCI6InBhbmd1eWVuQHBhbG9hbHRvbmV0d29ya3MuY29tIiwianRpIjoiNTlaM01WTUdJZyIsInNjb3BlIjp7ImJhbmZmLWVudGVycHJpc2UtZGVtbyI6eyJhZG1pbiI6dHJ1ZSwicm9sZVNvdXJjZSI6IkFEIEdyb3VwIn19LCJpc2xvY2tlZCI6ZmFsc2UsInRlbmFudGlkIjoiYmFuZmYtZW50ZXJwcmlzZS1kZW1vIiwidXJsUGF0dGVybiI6Ii9wdWIvdjQuMC8iLCJ0eXBlIjoiZGV2aWNlX3JldHJpZXZhbF9rZXkiLCJpYXQiOjE2MDU3NjQ5NDQsImV4cCI6MTkyMTEyNDk0NCwiaXNzIjoiemluZ2JveCJ9._PE_XztIsin4w1nKAlcS3ZJdMYBAQUSH5cF71-ZC0EI"}], "data_inputs_options": [{"type": "customized_var", "name": "customer_id", "title": "Customer ID", "description": "", "required_on_edit": false, "required_on_create": true, "format_type": "text", "default_value": "", "placeholder": ""}, {"type": "customized_var", "name": "access_key_id", "title": "Access Key ID", "description": "", "required_on_edit": false, "required_on_create": true, "format_type": "password", "default_value": "", "placeholder": ""}, {"type": "customized_var", "name": "secret_access_key", "title": "Secret Access Key", "description": "", "required_on_edit": false, "required_on_create": true, "format_type": "password", "default_value": "", "placeholder": ""}], "code": "\n# encoding = utf-8\n\nimport os\nimport sys\nimport time\nimport datetime\nimport json\n\n'''\n    IMPORTANT\n    Edit only the validate_input and collect_events functions.\n    Do not edit any other part in this file.\n    This file is generated only once when creating the modular input.\n'''\n'''\n# For advanced users, if you want to create single instance mod input, uncomment this method.\ndef use_single_instance_mode():\n    return True\n'''\n\ndef validate_input(helper, definition):\n    \"\"\"Implement your own validation logic to validate the input stanza configurations\"\"\"\n    # This example accesses the modular input variable\n    customer_id = definition.parameters.get('customer_id', None)\n    access_key_id = definition.parameters.get('access_key_id', None)\n    secret_access_key = definition.parameters.get('secret_access_key', None)\n    pass\n\ndef query_api(helper, url, parameters, api_type, proxy_enabled):\n    global_page_length = 1000\n    total = 1000\n    results = []\n    start_time = time.time()\n    page_offset = 0\n    \n    if api_type == 'devices':\n        items = 'devices'\n        page_offset = helper.get_check_point(\"offset\")\n        if not page_offset:\n            page_offset = 1000\n        page = 0\n        max_pages = 20\n        \n        while page < max_pages:\n            method = 'GET'\n            response = helper.send_http_request(url, method, parameters,\n                                                payload=None, headers=None,\n                                                cookies=None, verify=True, cert=None,\n                                                timeout=30, use_proxy=proxy_enabled)\n            r_status = response.status_code\n            if r_status == 200:\n                entries = response.json()[items]\n                results = results + entries\n                total = len(entries)\n                page_offset = page_offset + global_page_length\n                parameters.update({'offset': page_offset})\n                page += 1\n                helper.log_debug(\"Current Offset: {0}, Total Entries: {1}, Next Page: {2}\".format(page_offset, total, page) )\n                if total < global_page_length:\n                    helper.delete_check_point(\"offset\")\n                    helper.delete_check_point(\"last_run_end\")\n                    helper.log_debug(\"End of device list. Cleared checkpoint data.\")\n                    break\n            else:\n                helper.log_debug(r_status)\n                break\n        else: \n            now = datetime.datetime.now()\n            helper.save_check_point(\"offset\", page_offset)\n            helper.save_check_point(\"last_run_timestamp\", datetime.datetime.strftime(now, \"%Y-%m-%d %H:%M:%S\"))\n            helper.log_debug(\"We have reached max_page. Saved offset: {0} last_run_end: {1}\".format(page_offset, now))\n\n            \n    else:\n        items = 'items'\n        while total == global_page_length:\n            method = 'GET'\n            response = helper.send_http_request(url, method, parameters,\n                                                payload=None, headers=None,\n                                                cookies=None, verify=True, cert=None,\n                                                timeout=30, use_proxy=proxy_enabled)\n            r_status = response.status_code\n            if r_status == 200:\n                entries = response.json()[items]\n                results = results + entries\n                total = len(entries)\n                page_offset = page_offset + global_page_length\n                helper.log_debug(\"Current Offset: {0}, Total Entries: {1}\".format(page_offset, total) )\n                parameters.update({'offset': page_offset})\n            else:\n                helper.log_debug(r_status)\n                break\n    run_time = time.time() - start_time\n    helper.log_debug(\"End of {0} results. Function took {1} to run\".format(api_type, run_time))\n    return (results)\n\ndef collect_events(helper, ew):\n    # Set debug level\n    log_level = helper.get_log_level()\n    helper.set_log_level(log_level)\n    # Get Proxy Settings\n    proxy_settings = helper.get_proxy()\n    proxy_enabled = bool(proxy_settings)\n    # helper.log_debug(\"Checking if Proxy is enabled\")\n    # helper.log_debug(proxy_enabled) \n\n    opt_customer_id = helper.get_arg('customer_id')\n    opt_access_key_id = helper.get_arg('access_key_id')\n    opt_secret_access_key = helper.get_arg('secret_access_key')\n\n    global_url = \"https://{0}.iot.paloaltonetworks.com/pub/v4.0\".format(\n        opt_customer_id)\n    global_url_params = {\n        'customerid': opt_customer_id,\n        'key_id': opt_access_key_id,\n        'access_key': opt_secret_access_key,\n        'pagelength': 1000,\n        'offset': 0,\n    }\n\n    last_device_pull = helper.get_check_point(\"last_run_timestamp\")\n\n    if not last_device_pull or datetime.datetime.strptime(last_device_pull, \"%Y-%m-%d %H:%M:%S\") < datetime.datetime.now() - datetime.timedelta(minutes=5):\n        # Lets get Device Inventory\n        try:\n            device_url = '{0}/device/list'.format(global_url)\n            params = {\n                'filter_monitored': 'yes',\n                'detail': 'true',\n            }\n            params.update(global_url_params)\n            devices = query_api(helper, device_url, params, 'devices', proxy_enabled)\n            for data in devices:\n                try:\n                    event = helper.new_event(\n                        host=global_url,\n                        source=helper.get_input_stanza_names(),\n                        index=helper.get_output_index(),\n                        sourcetype='pan:iot_device',\n                        data=json.dumps(data))\n                    ew.write_event(event)\n                except Exception as e:\n                    helper.log_error('Error on parse event. ' + str(e))\n        except Exception as e:\n            print(str(e))\n    else:\n        helper.log_debug(\"Skipping device inventory pull. Last pulled: {0}\".format(last_device_pull))\n\n    # Lets get Alerts\n    try: \n        alerts_url = '{0}/alert/list'.format(global_url)\n        params = {\n            'type': 'policy_alert',\n        }\n        params.update(global_url_params)\n        alerts = query_api(helper, alerts_url, params, 'alerts', proxy_enabled)\n        for data in alerts:\n            try:\n                event = helper.new_event(\n                    host=global_url,\n                    source=helper.get_input_stanza_names(),\n                    index=helper.get_output_index(),\n                    sourcetype='pan:iot_alert',\n                    data=json.dumps(data))\n                ew.write_event(event)\n            except Exception as e:\n                helper.log_error('Error on parse event. ' + str(e))\n    except Exception as e:\n        helper.log_error(str(e))\n\n    # # Vulnerabilities\n    try:\n        vuln_url = '{0}/vulnerability/list'.format(global_url)\n        params = {\n            'groupby': 'device',\n        }\n        params.update(global_url_params)\n        vulnerabilities = query_api(helper, vuln_url, params, 'vulnerabilities', proxy_enabled)\n        for data in vulnerabilities:\n            try:\n                event = helper.new_event(\n                    host=global_url,\n                    source=helper.get_input_stanza_names(),\n                    index=helper.get_output_index(),\n                    sourcetype='pan:iot_vulnerability',\n                    data=json.dumps(data))\n                ew.write_event(event)\n            except Exception as e:\n                helper.log_error('Error on parse event. ' + str(e))\n    except Exception as e:\n        helper.log_error(str(e))\n", "customized_options": [{"name": "customer_id", "value": "banff-enterprise-demo"}], "uuid": "633c7b1243dc44178b70dd5e260fbd80", "sample_count": "10008"}, {"index": "default", "sourcetype": "minemeld_feed", "interval": "30", "use_external_validation": true, "streaming_mode_xml": true, "name": "minemeld_feed", "title": "MineMeld Feed", "description": "", "type": "customized", "parameters": [{"name": "feed_url", "label": "  Output Node Feed URL2, "help_string": "https://<feed_url>", "required": true, "format_type": "text", "default_value": "", "placeholder": "", "type": "text", "value": ""}, {"name": "indicator_timeout", "label": "Indicator Timeout", "help_string": " How long to retain indicators (in hours)", "required": true, "format_type": "text", "default_value": "", "placeholder": "", "type": "text", "value": ""}, {"name": "credentials", "label": "Feed Credentials", "help_string": "", "required": false, "possible_values": [], "format_type": "global_account", "default_value": "", "placeholder": "", "type": "global_account", "value": ""}], "data_inputs_options": [{"type": "customized_var", "name": "feed_url", "title": "  Output Node Feed URL2", "description": "https://<feed_url>", "required_on_edit": false, "required_on_create": true, "format_type": "text", "default_value": "", "placeholder": ""}, {"type": "customized_var", "name": "indicator_timeout", "title": "Indicator Timeout", "description": " How long to retain indicators (in hours)", "required_on_edit": false, "required_on_create": true, "format_type": "text", "default_value": "", "placeholder": ""}, {"type": "customized_var", "name": "credentials", "title": "Feed Credentials", "description": "", "required_on_edit": false, "required_on_create": false, "possible_values": [], "format_type": "global_account", "default_value": "", "placeholder": ""}], "code": "\n# encoding = utf-8\n\nimport base64\nimport functools\nimport json\nimport os\nimport requests.exceptions\nimport sys\nimport time\n\nVERIFY_CERTIFICATE = True\n\n'''\n    IMPORTANT\n    Edit only the validate_input and collect_events functions.\n    Do not edit any other part in this file.\n    This file is generated only once when creating the modular input.\n'''\n'''\n# For advanced users, if you want to create single instance mod input, uncomment this method.\ndef use_single_instance_mode():\n    return True\n'''\n\ndef timer(desc):\n    def outer(func):\n        @functools.wraps(func)\n        def inner(*args):\n            \"\"\"Decorator to time function execution.\n\n            If an exception is raised during the function, then a time of \"-1\"\n            will be saved for the given description.\n\n            Note: Any function decorated with this should have the \"stats\" dict\n            as the final argument in its arg list.\n\n            \"\"\"\n            # Setup.\n            stats = args[-1]\n            stats[desc] = -1\n            start = time.time()\n\n            # Execute the function.\n            ret_val = func(*args)\n\n            # No exception, so save the runtime and return ret_val.\n            stats[desc] = time.time() - start\n            return ret_val\n        return inner\n    return outer\n\n\ndef validate_input(helper, definition):\n    \"\"\"Implement your own validation logic to validate the input stanza configurations\"\"\"\n    # This example accesses the modular input variable\n    # feed_url = definition.parameters.get('feed_url', None)\n    # credentials = definition.parameters.get('credentials', None)\n    pass\n\n\ndef collect_events(helper, ew):\n    \"\"\"Collect the kvstore events from the feed.\"\"\"\n    # Get the short name for this feed.\n    name = helper.get_input_stanza_names()\n    start = time.time()\n    try:\n        indicator_timeout = int(helper.get_arg('indicator_timeout')) * 3600\n    except ValueError:\n        # If this isn't set, timeout indicators immediately.\n        indicator_timeout = 0\n    stats = {'input_name': name}\n\n    helper.log_info('START Splunk_TA_paloalto indicator retrieval for \"{0}\"'.format(\n        name))\n\n    # Get the current indicators.\n    kvs_entries = pull_from_kvstore(helper, name, start, stats)\n    stats['previous_indicators'] = len(kvs_entries)\n\n    # Retrieve current entries from the MineMeld feed.\n    mmf_entries = []\n    try:\n        mmf_entries = get_feed_entries(helper, name, start, stats)\n    except requests.exceptions.HTTPError as e:\n        helper.log_error('Failed to get entries for \"{0}\": {1}'.format(\n            name, e))\n        stats['error'] = str(e)\n    stats['feed_indicators'] = len(mmf_entries)\n\n    # Merge the two together, and determine which indicators should be expired.\n    rm_entries, retained_indicators = merge_entries(\n        mmf_entries, kvs_entries, start, indicator_timeout, stats)\n    stats['expired_indicators'] = len(rm_entries)\n    stats['indicators'] = len(mmf_entries) + retained_indicators\n\n    # Save new/updated indicators to the kvstore.\n    save_to_kvstore(helper, name, mmf_entries, stats)\n\n    # Delete the expired indicators.\n    remove_from_kvstore(helper, name, rm_entries, stats)\n\n    # Write an event to the index giving some basic stats.\n    stats['total_time'] = time.time() - start\n    save_stats_as_event(helper, ew, stats)\n\n    # Done\n    helper.log_info('END Splunk_TA_paloalto indicator retrieval for \"{0}\"'.format(\n        name))\n\n\n@timer('read_kvstore')\ndef pull_from_kvstore(helper, name, start, stats):\n    \"\"\"Retrieves all current indicators.\"\"\"\n    resp = helper.send_http_request(\n        url=_uri(helper),\n        headers=_headers(helper),\n        method='GET',\n        verify=False,\n        parameters={'query': json.dumps({'splunk_source': name})})\n    resp.raise_for_status()\n\n    ans = {}\n    for v in resp.json():\n        ans[v['indicator']] = {\n            '_key': v['_key'],\n            'is_present': False,\n            'splunk_last_seen': v.get('splunk_last_seen', 0.0)}\n\n    return ans\n\n\n@timer('retrieve_indicators')\ndef get_feed_entries(helper, name, start, stats):\n    \"\"\"Pulls the indicators from the minemeld feed.\"\"\"\n    feed_url = helper.get_arg('feed_url')\n    feed_creds = helper.get_arg('credentials')\n    feed_headers = {}\n    # If auth is specified, add it as a header.\n    if feed_creds is not None:\n        auth = '{0}:{1}'.format(feed_creds['username'], feed_creds['password']).encode('ascii')\n        auth = base64.b64encode(auth)\n        auth = auth.decode('utf-8')\n        feed_headers['Authorization'] = 'Basic {0}'.format(auth)\n\n    # Pull events as json.\n    resp = helper.send_http_request(\n        url=feed_url,\n        method='GET',\n        parameters={'v': 'json', 'tr': 1},\n        headers=feed_headers,\n        verify=VERIFY_CERTIFICATE,\n    )\n\n    # Raise exceptions on problems.\n    resp.raise_for_status()\n    feed_entries = resp.json()\n\n    # Return the normalized events to be saved to the kv store.\n    return normalized(name, feed_entries, start)\n\n\n@timer('merge_indicators')\ndef merge_entries(mmf_entries, kvs_entries, start, indicator_timeout, stats):\n    \"\"\"\n    Merges the current indicators with previous, determining which should\n    be expired.\n    \"\"\"\n    rm_entries = []\n    retained_indicators = 0\n\n    for mmfe in mmf_entries:\n        kvse = kvs_entries.get(mmfe['indicator'])\n        if kvse is not None:\n            kvse['is_present'] = True\n            mmfe['_key'] = kvse['_key']\n\n    for info in iter(kvs_entries.values()):\n        if info['is_present']:\n            pass\n        elif info['splunk_last_seen'] + indicator_timeout < start:\n            rm_entries.append(info['_key'])\n        else:\n            retained_indicators += 1\n\n    return rm_entries, retained_indicators\n\n\n@timer('save_to_kvstore')\ndef save_to_kvstore(helper, name, entries, stats):\n    \"\"\"Saves all normalized entries as `name` events.\"\"\"\n    helper.log_info('Saving {0} entries for MineMeld feed \"{1}\"'.format(\n        len(entries), name))\n    url = '{0}/batch_save'.format(_uri(helper))\n\n    # We need to batch in groups of 500, the default.\n    for i in range(0, len(entries), 500):\n        resp = helper.send_http_request(\n            url=url,\n            headers=_headers(helper),\n            method='POST',\n            verify=False,\n            payload=entries[i:i+500])\n        resp.raise_for_status()\n\n\n@timer('remove_from_kvstore')\ndef remove_from_kvstore(helper, name, rm_entries, stats):\n    \"\"\"Removes the specified entries from the kvstore.\"\"\"\n    if not rm_entries:\n        return\n\n    helper.log_info('Removing {0} kvstore entries for MineMeld feed \"{1}\"'.format(\n        len(rm_entries), name))\n    url = _uri(helper)\n    headers = _headers(helper)\n\n    # Batch a few at a time, as splunk 414s if the URI is too long, or times\n    # out if it's within the length limits but still hits too many entries to\n    # finish on time.  From some tests, it seems like 500 is a good number,\n    # which is nice since it matches the batch_save number.\n    #\n    # The _key field has been 24 characters in length on my system.\n    for i in range(0, len(rm_entries), 500):\n        rms = rm_entries[i:i+500]\n        query = {'$or': list({'_key': x} for x in rms)}\n        resp = helper.send_http_request(\n            url=url,\n            headers=headers,\n            method='DELETE',\n            verify=False,\n            parameters={'query': json.dumps(query)})\n        resp.raise_for_status()\n\n\ndef save_stats_as_event(helper, ew, stats):\n    \"\"\"Saves the stats of getting feed events to the index.\"\"\"\n    event = helper.new_event(\n        source=helper.get_input_type(),\n        index=helper.get_output_index(),\n        sourcetype=helper.get_sourcetype(),\n        data=json.dumps(stats),\n    )\n    ew.write_event(event)\n\n\ndef _uri(helper):\n    \"\"\"Returns the URL of the kvstore.\"\"\"\n    return '/'.join((\n        helper.context_meta['server_uri'],\n        'servicesNS',\n        'nobody',\n        'Splunk_TA_paloalto',\n        'storage',\n        'collections',\n        'data',\n        'minemeldfeeds'))\n\n\ndef _headers(helper):\n    \"\"\"Returns the auth header for Splunk.\"\"\"\n    return {\n        'Authorization': 'Splunk {0}'.format(\n            helper.context_meta['session_key'])}\n\n\ndef normalized(name, feed_entries, start):\n    \"\"\"Returns a list of normalized kvstore entries.\"\"\"\n    data = []\n    for feed_entry in feed_entries:\n        if 'indicator' not in feed_entry or 'value' not in feed_entry:\n            continue\n\n        # Make the entry dict.\n        entry = feed_entry.copy()\n        entry['splunk_source'] = name\n        entry['splunk_last_seen'] = start\n\n        data.append(entry)\n\n    return data\n", "customized_options": [{"name": "feed_url", "value": ""}, {"name": "indicator_timeout", "value": ""}, {"name": "credentials", "value": ""}], "uuid": "81937edbc4ef44a89fb8041f1c1f1624", "sample_count": 0}]}, "field_extraction_builder": {"pan:aperture": {"data_format": "json"}, "pan:config": {"data_format": "tabular", "table_results": {"delim": ","}}, "pan:decryption": {"data_format": "tabular", "table_results": {"delim": ","}}, "pan:globalprotect": {"data_format": "tabular", "table_results": {"delim": ","}}, "pan:log": {"data_format": "tabular", "table_results": {"delim": ","}}, "pan:minemeld": {"data_format": "json"}, "pan:system": {"data_format": "tabular", "table_results": {"delim": ","}}, "pan:threat": {"data_format": "tabular", "table_results": {"delim": ","}}, "pan:traffic": {"data_format": "tabular", "table_results": {"delim": ","}}, "pan:userid": {"data_format": "tabular", "table_results": {"delim": ","}}, "pan:iot": {"data_format": "json"}, "pan:iot_alert": {"data_format": "json"}, "pan:iot_device": {"data_format": "json"}, "pan:iot_vulnerability": {"data_format": "json"}, "pan:firewall_cloud": {"data_format": "tabular", "table_results": {"delim": " "}}, "pan:xdr_incident": {"data_format": "json"}}, "global_settings_builder": {"global_settings": {"proxy_settings": {"proxy_type": "http"}, "log_settings": {}, "credential_settings": [], "customized_settings": [{"required": false, "name": "autofocus_api_key", "label": "  AutoFocus API Key", "placeholder": "", "default_value": "", "help_string": " Used to retrieve metadata about AutoFocus tags. Requires a MineMeld Feed input to download threat indicators. More info: https://splunk.paloaltonetworks.com/autofocus-and-minemeld.html", "type": "password", "format_type": "password", "value": ""}, {"required": false, "name": "wildfire_api_key", "label": "  WildFire API Key", "placeholder": "", "default_value": "", "help_string": " Used to retrieve reports from the WildFire Cloud. An API Key is available from the WildFire Portal. https://wildfire.paloaltonetworks.com", "type": "password", "format_type": "password", "value": ""}]}}, "sourcetype_builder": {"minemeld_feed": {"metadata": {"data_input_name": "minemeld_feed"}}}, "validation": {}}