Wider edl checkpoint support

.gitignore

@@ -124,3 +124,4 @@ venv.bak/
 .idea/**/uiDesigner.xml
 .idea/**/dbnavigator.xml
 
+.idea

Pipfile

@@ -0,0 +1,13 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+irflow-integrations = {editable = true,path = "./../irflow-integrations"}
+
+[packages]
+flask="*"
+
+[requires]
+python_version = "3.6"

integrations_websvc/__init__.py

@@ -0,0 +1,20 @@
+""" Simple Static File Server for use with IR-Flow and the IR-Flow Integrations Framework """
+from flask import Flask, send_from_directory
+import os
+
+from integrations_websvc import config
+
+app = Flask(__name__)
+
+app.config.from_object(config)
+if os.path.exists(os.path.join(os.getcwd(), 'config.py')):
+    app.config.from_pyfile(os.path.join(os.getcwd(), 'config.py'))
+
+app.secret_key = app.config['SECRET_KEY']
+
+STATIC_FILES_PATH = app.config['BASE_PATH']
+
+
+@app.route('/lists/<path:path>')
+def get_list_at_path(path):
+    return send_from_directory(STATIC_FILES_PATH, path)

integrations_websvc/config.py

@@ -0,0 +1,10 @@
+import secrets
+import os
+from os import environ as env
+
+APP_NAME='irflow_integrations_websvc'
+DEBUG = True if env.get('INTEGRATIONS_DEBUG', 'false').lower() == 'true' else False
+IP = env.get('INTEGRATIONS_WEBSVC_IP', '0.0.0.0')
+PORT = env.get('INTEGRATIONS_WEBSVC_PORT', 8080)
+SECRET_KEY = env.get('INTEGRATIONS_WEBSVC_SECRET_KEY', default=''.join(secrets.token_hex(16)))
+BASE_PATH = env.get('INTEGRATIONS_WEBSVC_BASE_PATH', os.path.abspath('integrations_websvc/static/'))

integrations_websvc/static/checkpoint/test_multi.csv

@@ -0,0 +1,3 @@
+domain,www.ynet.co.il/articles/,testname1,Ynet articles Access
+url,www.ynet.co.il/articles/,testname2,Ynet articles Access
+md5,f58628917abcbcfb2b2258b6b46bf721,testname3,Random bad hash # comment about this entry

irflow-integrations-websvc.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Integrations Webservice to Host Files that enable EDL, CheckPoint, etc.
+After=network.target
+
+[Service]
+User=irflow
+Group=irflow
+WorkingDirectory=/home/irflow/irflow-integration-websvc
+ExecStart=/bin/pipenv run ./wsgi.py
+
+[Install]
+WantedBy=multi-user.target

readme.rst

@@ -1,6 +1,78 @@
-Simple Flask Webserver to serve text files
+Integrations Webservice
+=======================
 
+This repo contains the source for the Integrations Framework Webservice that presently provides support for integrations
+that rely on a remote static file for dynamically ingesting threat information.
 
-To start:
+Installing the webservice
+-------------------------
 
-`python -m flask run --port=8081 --host=0.0.0.0`
+This repository should be cloned at the same level as your irflow-integrations repository:
+
+.. code-block:: shell
+
+    .
+    └── Syncurity
+        ├── irflow-integrations
+        └── irflow-integration-websvc
+
+From here, move into your newly cloned repo, and run the following (**Do not run as root**):
+
+.. code-block:: shell
+
+    $ ./setup_websvc.sh
+
+Once the service has been setup once, it can be started and stopped on demand with the ``start_websvc.sh`` and
+``stop_websvc.sh`` scripts respectively.
+
+Setting up for Development
+--------------------------
+
+Make sure that you have Pipenv_ installed. This is done automatically at the user level in production - install Pipenv
+as recommended on your system (generally ``pip install --user pipenv``). The project can be installed as developer with
+the following:
+
+.. code-block:: shell
+
+    $ pipenv install --dev --skip-lock
+
+
+Setting up your environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Pipenv supports the use of ``.env`` files. If your development configuration is unusual, you may need to provide default
+values for environment variables that are different from the assumed defaults. At minimum, a debug variable should be
+set when developing so that flask will provide debug information in-browser should an exception be hit:
+
+.. code-block:: shell
+
+    INTEGRATIONS_DEBUG=TRUE
+
+The above line should be place in a file named ``.env`` (with no prefix before the ``.``), which should be located in
+the root of the repository.
+
+Running the webservice
+^^^^^^^^^^^^^^^^^^^^^^
+
+Using pipenv, you can drop into a subshell within the pipenv-created environment by running
+
+.. code-block:: shell
+
+    $ pipenv shell
+
+Single commands can be run in the environment (without creating a subshell) with
+
+.. code-block:: shell
+
+    $ pipenv run <command>
+
+Once in the environment, the webservice can be run as follows
+
+.. code-block:: shell
+
+    $ ./wsgi.py
+
+
+
+
+.. _Pipenv: https://pipenv.readthedocs.io/en/latest/

setup_websvc.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+if [[ $EUID -eq 0 ]]; then
+    echo "This script should not be run as root!"
+    exit 1
+fi
+
+echo "===> Ensuring pipenv is installed"
+pip3 install -q --user pipenv --upgrade
+echo "===> Getting your environment set up"
+pipenv install > /dev/null
+echo "===> Moving Unit files as needed"
+
+if [ ! -f /usr/lib/systemd/system/irflow-integrations-websvc.service ] ; then
+    sudo cp irflow-integrations-websvc.service /usr/lib/systemd/system/
+    sudo systemctl daemon-reload
+    echo "===> Starting service"
+    ./start_websvc.sh
+else
+    echo "===> Service already set up, nothing to do!"
+    exit 1
+fi

start_websvc.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+systemctl is-active --quiet irflow-integrations-websvc
+retVal=$?
+if [[ ${retVal} -ne 0 ]]; then
+    echo "===> Starting Integrations Webservice"
+    sudo systemctl enable irflow-integrations-websvc
+    sudo systemctl start irflow-integrations-websvc
+else
+    echo "===> Service already running!"
+fi
+
+exit ${retVal}

stop_websvc.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+systemctl is-active --quiet irflow-integrations-websvc
+retVal=$?
+if [[ ${retVal} -ne 0 ]]; then
+    echo "===> Service already stopped!"
+else
+    echo "===> Stopping Integrations Webservice"
+    sudo systemctl stop irflow-integrations-websvc
+fi
+
+exit ${retVal}

wsgi.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+""" WSGI for the IR-Flow Integrations Webservice
+
+Example::
+
+    `$ ./wsgi.py`
+
+
+"""
+from integrations_websvc import app
+
+if __name__ == '__main__':
+    app.run(host=app.config['IP'], port=app.config['PORT'])
+
+application = app