Merge branch 'master' into phil-docs-update2

LICENSE2.md

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2019, Two Six Labs
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -44,3 +44,32 @@ See the various `dockerized-` commands in the `bin` directory for interacting wi
 
 
 `updated 2018-05-01T16:34:00EST`
+
+Copyright 2019 Two Six Labs
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Savior Project GPL 2.0
+Copyright (C) 2019 Two Six Labs
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

bin/bootstrap.ps1

@@ -5,21 +5,12 @@
 # To run this script, copy and paste the text between <# and #> Below
 <# powershell -NoProfile -ExecutionPolicy ByPass -File .\bootstrap.ps1 #>
 
-Write-Output "Downloading python 3.6.4 and git"
+Write-Output "Downloading git"
 
 mkdir c:\temp
 
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
 
-Write-Output "Download and Install python . . . "
-Invoke-WebRequest -Uri "https://www.python.org/ftp/python/3.6.4/python-3.6.4.exe" -OutFile c:\TEMP\python-3.6.4.exe
-Write-Output "Installing Python .  . ."
-Start-Process -FilePath C:\Windows\System32\msiexec.exe -Wait -ArgumentList "/i", "c:\TEMP\python-3.6.4.exe", "/quiet",  "TARGETDIR=c:\Python-3.6.4", "InstallAllUsers=1", "PrependPath=1"
-Write-Output "Removing Python installer .  . ."
-Remove-Item c:\TEMP\python-3.6.4.exe -Force
-$Env:PATH="c:\Python-3.6.4\Scripts;c:\Python-3.6.4;" + $Env:PATH
-python -m pip install --upgrade pip
-
 Write-Output "Downloading git . . . "
 Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.16.2.windows.1/Git-2.16.2-64-bit.exe" -OutFile c:\TEMP\Git-2.16.2-64-bit.exe
 Write-Output "Installing git .  . ."
@@ -28,9 +19,8 @@ Write-Output "Removing git installer .  . ."
 Remove-Item c:\TEMP\Git-2.16.2-64-bit.exe -Force
 $Env:PATH="C:\Program Files\Git\cmd;" + $Env:PATH
 
-Write-Output "Clone, fetch and checkout savior . . . "
+Write-Output "Clone savior repo . . . "
 git clone -v https://github.com/twosixlabs/savior.git
 cd ./savior
-git fetch -v origin
 
 

control/api_server/lib/extraction_plug.ex

@@ -371,8 +371,9 @@ defmodule ApiServer.ExtractionPlug do
   end
 
   # Is the given string a valid Virtue ID?
+  # Per NC, a Virtue ID could be a FQDN, so this check is quite permissive
   def is_virtue_id(st) do
-    is_uuid?(st)
+    String.match?(st, ~r/^[a-zA-Z0-9\.\-_]+$/)
   end
 
   # Is the given string a valid Application ID?

control/api_server/lib/targeting_utils.ex

@@ -42,8 +42,8 @@ defmodule ApiServer.TargetingUtils do
 
     targeting =
       raw_targeting
-      |> rename_key("virtue", "virtue_id")
-      |> rename_key("sensor", "sensor_id")
+      |> rename_key(:virtue, :virtue_id)
+      |> rename_key(:sensor, :sensor_id)
 
 
     case ApiServer.Sensor.get_many(targeting) do
@@ -61,7 +61,7 @@ defmodule ApiServer.TargetingUtils do
       true ->
         v = Map.get(map, old_key)
         map
-        |> Map.drop(old_key)
+        |> Map.delete(old_key)
         |> Map.put(new_key, v)
       false ->
         map

control/logging/config/server.properties

@@ -121,7 +121,7 @@ transaction.state.log.min.isr=1
 # from the end of the log.
 
 # The minimum age of a log file to be eligible for deletion due to age
-log.retention.hours=168
+log.retention.hours=12
 
 # A size-based retention policy for logs. Segments are pruned from the log unless the remaining
 # segments drop below log.retention.bytes. Functions independently of log.retention.hours.
@@ -154,4 +154,4 @@ zookeeper.connection.timeout.ms=6000
 # The default value for this is 3 seconds.
 # We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
 # However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
-group.initial.rebalance.delay.ms=0
+group.initial.rebalance.delay.ms=0

control/virtue-security/run-local.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+export CFSSL_SHARED_SECRET=de1069ab43f7f385d9a31b76af27e7620e9aa2ad5dccd264367422a452aba67f
+
+mkdir certs
+
+echo "Getting Client Certificate"
+python ./tools/get_certificates.py --cfssl-host sensing-ca.savior.internal --hostname api -d certs --quiet
+
+echo "Running virtue-security"
+python virtue-security --public-key-path certs/cert.pem "$@" --ca-key-path certs/ca.pem --private-key-path certs/cert-key.pem

control/virtue-security/virtue-security

@@ -3,6 +3,7 @@ __VERSION__ = "1"
 
 from aiohttp import web, ClientSession
 from aiohttp.client_exceptions import ClientConnectionError, ClientPayloadError
+import urllib
 import argparse
 import asyncio
 import base64
@@ -4361,29 +4362,36 @@ async def api_monitor(opts):
         print("   ! status_code(%d): %s" % (res.status_code, http_code_to_message(res.status_code)))
 
 
-async def api_stream(opts):
+def api_stream_sync(opts):
     """
     Retrieve the JSONL stream from the server and print it out.
 
     :param opts:
     :return:
+
+    This functions synchronously for speed. A Python coroutine cannot
+    keep up with the data.
     """
 
+    secure = False
+    print("% virtue-security api_stream(): preferred method of streaming logs is")
+    print("%   curl http://sensing-api.savior.internal:17141/api/v1/sensors/all/stream")
+
     # build up our parameters
     since = since_datetime(opts.time_since)
     follow = opts.log_follow
     log_level = opts.log_level
 
-    full_uri = construct_api_uri(opts, "/sensor/%s/stream" % (opts.var_sensor,))
+    full_uri = construct_api_uri(opts, "/sensor/%s/stream" % (opts.var_sensor,), secure)
 
     if opts.all_sensors:
-        full_uri = construct_api_uri(opts, "/sensors/all/stream")
+        full_uri = construct_api_uri(opts, "/sensors/all/stream", secure)
 
     elif opts.var_username is not None:
-        full_uri = construct_api_uri(opts, "/user/%s/stream" % (opts.var_username,))
+        full_uri = construct_api_uri(opts, "/user/%s/stream" % (opts.var_username,), secure)
 
     elif opts.var_sensor is not None:
-        full_uri = construct_api_uri(opts, "/sensor/%s/stream" % (opts.var_sensor,))
+        full_uri = construct_api_uri(opts, "/sensor/%s/stream" % (opts.var_sensor,), secure)
 
     get_params = {
         "follow": str(follow),
@@ -4408,40 +4416,47 @@ async def api_stream(opts):
     print("% virtue-security api_stream(uri={})\n".format(full_uri))
 
     # spin up our request and start streaming
-    ssl_context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
-    ssl_context.load_verify_locations(cafile=opts.ca_key_path)
-    ssl_context.load_cert_chain(os.path.abspath(opts.public_key_path), os.path.abspath(opts.private_key_path))
+    if secure:
+        ssl_context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
+        ssl_context.load_verify_locations(cafile=opts.ca_key_path)
+        ssl_context.load_cert_chain(os.path.abspath(opts.public_key_path), os.path.abspath(opts.private_key_path))
+    else:
+        print("% virtue-security api_stream(): using insecure connection to API server")
+        ssl_context = None
 
-    async with ClientSession(read_timeout=None) as session:
-        async with session.get(full_uri, params=get_params, ssl_context=ssl_context) as response:
+    request = urllib.request.Request(full_uri, headers=get_params, method='GET')
 
-            # this is an async stream reader, so we pull in byte arrays
-            chunked_binary = b''
-            try:
-                while True:
-                    chunk = await response.content.read(1024)
-                    if not chunk:
-                        if len(chunked_binary) > 0:
-                            manage_stream_output(opts, chunked_binary.decode("utf-8"), logger)
-                        break
+    with urllib.request.urlopen(request, context=ssl_context) as response:
+        chunked_binary = b''
+        try:
+            while True:
+                chunk = response.read(16*1024)
+                if not chunk:
+                    if len(chunked_binary) > 0:
+                        manage_stream_output(opts, chunked_binary.decode("utf-8"), logger)
+                    break
 
-                    chunked_binary += chunk
+                chunked_binary += chunk
 
-                    # try and parse out a full line up to a newline
-                    msg, chunked_binary = extract_jsonl_line(chunked_binary)
+                # try and parse out a full line up to a newline
+                msg, chunked_binary = extract_jsonl_line(chunked_binary)
 
-                    # if we have a message, let's do something with it
-                    if msg is not None:
-                        manage_stream_output(opts, msg, logger)
+                # if we have a message, let's do something with it
+                if msg is not None:
+                    manage_stream_output(opts, msg, logger)
 
-            except ClientConnectionError as cce:
-                # print("caught aiohttp ClientConnectionError")
-                #
-                # We're eating this error because in certain circumstances
-                # the aiohttp library decides to throw an exception when
-                # the remote stream (in streaming situations) closes, rather
-                # than just EOF the stream reader. Because WFT.
-                print(cce)
+        except ClientConnectionError as cce:
+            # print("caught aiohttp ClientConnectionError")
+            #
+            # We're eating this error because in certain circumstances
+            # the aiohttp library decides to throw an exception when
+            # the remote stream (in streaming situations) closes, rather
+            # than just EOF the stream reader. Because WFT.
+            print(cce)
+
+
+async def api_stream(opts):
+    api_stream_sync(opts)
 
 
 def manage_stream_output(opts, stream_line, logger):

docker-compose-swarm.yml

@@ -22,7 +22,8 @@ services:
       CFSSL_SHARED_SECRET: "de1069ab43f7f385d9a31b76af27e7620e9aa2ad5dccd264367422a452aba67f"
       KAFKA_ADVERTISED_HOST_NAME: "sensing-kafka.savior.internal"      
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9455,OUTSIDE://$(KAFKA_ADVERTISED_HOST_NAME):9555
+      #KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9455,OUTSIDE://$(KAFKA_ADVERTISED_HOST_NAME):9555 # doesn't pull from previous environment variable...
+      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9455,OUTSIDE://sensing-kafka.savior.internal:9555
       KAFKA_LISTENERS: INSIDE://:9455,OUTSIDE://:9555
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:SSL,OUTSIDE:SSL
       KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE

sensors/ntfltmgr/ntfltmgr.py

@@ -1763,7 +1763,7 @@ def main(argv):
     test_command_response()
 
     while True:
-        curio.run(test_packet_decode, debug=[curio.debug.schedtrace,], with_monitor=True)
+        curio.run(test_packet_decode, with_monitor=True)
 
     sys.exit(0)