From b3e3bb7394d7bee9a60ff7a77af410ea20a731fe Mon Sep 17 00:00:00 2001
From: Travis <automateduser@example.com>
Date: Tue, 25 Sep 2018 17:13:27 +0000
Subject: [PATCH] [ci skip] Travis is updating the documentation; build no.:
 150

---
 docs/404.html                              |  42 ++++++++++-----------
 docs/authentication/index.html             |   2 +-
 docs/css/theme_extra.css                   |   5 ++-
 docs/index.html                            |   9 +++--
 docs/js/theme.js                           |  34 ++++++++++-------
 docs/modules/analyze/index.html            |   4 +-
 docs/modules/canonical_cve/index.html      |   2 +-
 docs/modules/collate/index.html            |   4 +-
 docs/modules/collection_archive/index.html |   2 +-
 docs/modules/collector/index.html          |   4 +-
 docs/modules/scheduler/index.html          |   6 +--
 docs/overview/index.html                   |   2 +-
 docs/search.html                           |   2 +-
 docs/search/main.js                        |  40 ++++++++++----------
 docs/search/search_index.json              |   2 +-
 docs/search/worker.js                      |   1 +
 docs/sitemap.xml                           |  18 ++++-----
 docs/sitemap.xml.gz                        | Bin 200 -> 202 bytes
 docs/swagger/jellyfish_swagger.yaml        |  12 +++---
 19 files changed, 104 insertions(+), 87 deletions(-)

diff --git a/docs/404.html b/docs/404.html
index 61b1e55..d0b3f02 100644
--- a/docs/404.html
+++ b/docs/404.html
@@ -7,16 +7,16 @@
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   
   
-  <link rel="shortcut icon" href="img/favicon.ico">
+  <link rel="shortcut icon" href="/img/favicon.ico">
   <title>Man o' War Documentation</title>
   <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
 
-  <link rel="stylesheet" href="css/theme.css" type="text/css" />
-  <link rel="stylesheet" href="css/theme_extra.css" type="text/css" />
+  <link rel="stylesheet" href="/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="/css/theme_extra.css" type="text/css" />
   <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
   
-  <script src="js/jquery-2.1.1.min.js" defer></script>
-  <script src="js/modernizr-2.8.3.min.js" defer></script>
+  <script src="/js/jquery-2.1.1.min.js" defer></script>
+  <script src="/js/modernizr-2.8.3.min.js" defer></script>
   <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
   <script>hljs.initHighlightingOnLoad();</script> 
   
@@ -29,9 +29,9 @@
     
     <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
       <div class="wy-side-nav-search">
-        <a href="." class="icon icon-home"> Man o' War Documentation</a>
+        <a href="/." class="icon icon-home"> Man o' War Documentation</a>
         <div role="search">
-  <form id ="rtd-search-form" class="wy-form" action="/search.html" method="get">
+  <form id ="rtd-search-form" class="wy-form" action="//search.html" method="get">
     <input type="text" name="q" placeholder="Search docs" title="Type search term here" />
   </form>
 </div>
@@ -43,17 +43,17 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="/.">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
 		
-    <a class="" href="authentication/">Authentication</a>
+    <a class="" href="/authentication/">Authentication</a>
 	    </li>
           
             <li class="toctree-l1">
 		
-    <a class="" href="overview/">Man o' War</a>
+    <a class="" href="/overview/">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
@@ -62,27 +62,27 @@
     <ul class="subnav">
                 <li class="">
                     
-    <a class="" href="modules/analyze/">Analyze</a>
+    <a class="" href="/modules/analyze/">Analyze</a>
                 </li>
                 <li class="">
                     
-    <a class="" href="modules/canonical_cve/">Canonical CVE</a>
+    <a class="" href="/modules/canonical_cve/">Canonical CVE</a>
                 </li>
                 <li class="">
                     
-    <a class="" href="modules/collate/">Collate</a>
+    <a class="" href="/modules/collate/">Collate</a>
                 </li>
                 <li class="">
                     
-    <a class="" href="modules/collection_archive/">Collection Archive</a>
+    <a class="" href="/modules/collection_archive/">Collection Archive</a>
                 </li>
                 <li class="">
                     
-    <a class="" href="modules/collector/">Collector</a>
+    <a class="" href="/modules/collector/">Collector</a>
                 </li>
                 <li class="">
                     
-    <a class="" href="modules/scheduler/">Scheduler</a>
+    <a class="" href="/modules/scheduler/">Scheduler</a>
                 </li>
     </ul>
 	    </li>
@@ -97,7 +97,7 @@
       
       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
         <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-        <a href=".">Man o' War Documentation</a>
+        <a href="/.">Man o' War Documentation</a>
       </nav>
 
       
@@ -105,7 +105,7 @@
         <div class="rst-content">
           <div role="navigation" aria-label="breadcrumbs navigation">
   <ul class="wy-breadcrumbs">
-    <li><a href=".">Docs</a> &raquo;</li>
+    <li><a href="/.">Docs</a> &raquo;</li>
     
     
     <li class="wy-breadcrumbs-aside">
@@ -152,9 +152,9 @@ <h1 id="404-page-not-found">404</h1>
       
     </span>
 </div>
-    <script>var base_url = '';</script>
-    <script src="js/theme.js" defer></script>
-      <script src="search/main.js" defer></script>
+    <script>var base_url = '/';</script>
+    <script src="/js/theme.js" defer></script>
+      <script src="/search/main.js" defer></script>
 
 </body>
 </html>
diff --git a/docs/authentication/index.html b/docs/authentication/index.html
index 084f0c4..7aae5e8 100644
--- a/docs/authentication/index.html
+++ b/docs/authentication/index.html
@@ -50,7 +50,7 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="..">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1 current">
diff --git a/docs/css/theme_extra.css b/docs/css/theme_extra.css
index cf8123e..ab107ba 100644
--- a/docs/css/theme_extra.css
+++ b/docs/css/theme_extra.css
@@ -128,8 +128,11 @@ pre .cs, pre .c {
  * Additions specific to the search functionality provided by MkDocs
  */
 
-.search-results article {
+.search-results {
     margin-top: 23px;
+}
+
+.search-results article {
     border-top: 1px solid #E1E4E5;
     padding-top: 24px;
 }
diff --git a/docs/index.html b/docs/index.html
index d69b5fe..51de810 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -50,7 +50,7 @@
           
             <li class="toctree-l1 current">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="current" href=".">Man o' War</a>
     <ul class="subnav">
             
     <li class="toctree-l2"><a href="#man-o-war">Man o' War</a></li>
@@ -147,6 +147,9 @@ <h2 id="introduction">Introduction</h2>
 <p>It was based of a project VDMS named called Jellyfish (it should be seen as an
 opensourced fork of the project). There are name references to it all over the
 place so keep that in mind.</p>
+<ul>
+<li><a href="https://github.com/chalbersma/manowar">Source Code</a></li>
+</ul>
               
             </div>
           </div>
@@ -194,6 +197,6 @@ <h2 id="introduction">Introduction</h2>
 </html>
 
 <!--
-MkDocs version : 1.0
-Build Date UTC : 2018-08-09 16:14:58
+MkDocs version : 1.0.4
+Build Date UTC : 2018-09-25 17:13:27
 -->
diff --git a/docs/js/theme.js b/docs/js/theme.js
index c3801dc..aecbb86 100644
--- a/docs/js/theme.js
+++ b/docs/js/theme.js
@@ -13,19 +13,27 @@ $( document ).ready(function() {
 
     // Keyboard navigation
     document.addEventListener("keydown", function(e) {
-        if ($(e.target).is(':input')) return true;
-        var key = e.which || e.keyCode || window.event && window.event.keyCode;
-        var page;
-        switch (key) {
-            case 78:  // n
-                page = $('[role="navigation"] a:contains(Next):first').prop('href');
-                break;
-            case 80:  // p
-                page = $('[role="navigation"] a:contains(Previous):first').prop('href');
-                break;
-            default: break;
-        }
-        if (page) window.location.href = page;
+      var key = e.which || e.keyCode || window.event && window.event.keyCode;
+      var page;
+      switch (key) {
+          case 78:  // n
+              page = $('[role="navigation"] a:contains(Next):first').prop('href');
+              break;
+          case 80:  // p
+              page = $('[role="navigation"] a:contains(Previous):first').prop('href');
+              break;
+          case 13:  // enter
+              if (e.target === document.getElementById('mkdocs-search-query')) {
+                e.preventDefault();
+              }
+              break;
+          default: break;
+      }
+      if ($(e.target).is(':input')) {
+        return true;
+      } else if (page) {
+        window.location.href = page;
+      }
     });
 
     $(document).on('click', "[data-toggle='rst-current-version']", function() {
diff --git a/docs/modules/analyze/index.html b/docs/modules/analyze/index.html
index acc9906..a888115 100644
--- a/docs/modules/analyze/index.html
+++ b/docs/modules/analyze/index.html
@@ -50,7 +50,7 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="../..">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
@@ -166,7 +166,7 @@ <h2 id="audits">Audits</h2>
 project. There you'll find some documentation on the content and structure of
 documentation that's available to you.</p>
 <h2 id="analyze-architecture">Analyze Architecture</h2>
-<p><img alt="Analyze Code Architecture" src="../../plantuml/analyze_diagram.svg" /></p>
+<p><img alt="Analyze Code Architecture" src="/plantuml/analyze_diagram.svg" /></p>
 <p>The general idea of analyze is quite simple. The process :</p>
 <ol>
 <li>Grabs a lists of hosts</li>
diff --git a/docs/modules/canonical_cve/index.html b/docs/modules/canonical_cve/index.html
index 62b1a2f..7237d8f 100644
--- a/docs/modules/canonical_cve/index.html
+++ b/docs/modules/canonical_cve/index.html
@@ -50,7 +50,7 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="../..">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
diff --git a/docs/modules/collate/index.html b/docs/modules/collate/index.html
index 99ebdb3..68dd1f8 100644
--- a/docs/modules/collate/index.html
+++ b/docs/modules/collate/index.html
@@ -50,7 +50,7 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="../..">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
@@ -153,7 +153,7 @@ <h2 id="introduction">Introduction</h2>
 This module does that. It figures passed, failed and exempt audits on a per-audit,
 per-pop and per-srvtype basis.</p>
 <h2 id="arch">Arch</h2>
-<p><img alt="Collate Architecture" src="../../plantuml/collate_diagram.svg" /></p>
+<p><img alt="Collate Architecture" src="/plantuml/collate_diagram.svg" /></p>
 <p>The idea here is simple. It will cycle through the various by factors (pop, srvtype
 audit/acoll). It counts up the results in <code>audits_by_host</code> for each of these
 factors and then update the related numbers in <code>audits_by_factor</code>.</p>
diff --git a/docs/modules/collection_archive/index.html b/docs/modules/collection_archive/index.html
index ff32507..dc061ae 100644
--- a/docs/modules/collection_archive/index.html
+++ b/docs/modules/collection_archive/index.html
@@ -50,7 +50,7 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="../..">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
diff --git a/docs/modules/collector/index.html b/docs/modules/collector/index.html
index 7eeb8f2..653b32e 100644
--- a/docs/modules/collector/index.html
+++ b/docs/modules/collector/index.html
@@ -50,7 +50,7 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="../..">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
@@ -151,7 +151,7 @@ <h1 id="collector">Collector</h1>
 to tell itself what it out to do. Then runs and parses those commands on individual
 hosts. It's main usage is to get called by <code>schedule2.py</code> over and over again to
 do the collections.</p>
-<p><img alt="Collector Arch" src="../../plantuml/collector_diagram.svg" /></p>
+<p><img alt="Collector Arch" src="/plantuml/collector_diagram.svg" /></p>
 <h2 id="collection-configuration">Collection Configuration</h2>
 <p>Collection configuration is by default stored in svn in the <code>collector.ini</code> file.
 Here is an example of a collection :</p>
diff --git a/docs/modules/scheduler/index.html b/docs/modules/scheduler/index.html
index 68b1f0f..0e4fce9 100644
--- a/docs/modules/scheduler/index.html
+++ b/docs/modules/scheduler/index.html
@@ -50,7 +50,7 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="../..">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
@@ -155,7 +155,7 @@ <h2 id="introduction">Introduction</h2>
 <p>The scheduler is a module that essentially interacts with three other modules,
 <code>collector</code>, <code>storage</code> and <code>sapicheck</code>. It ties these together and in a big ole
 loop enumerates the systems in Uber and attempts to profile them.</p>
-<p><img alt="Scheduler Highlevel" src="../../plantuml/scheduler_highlevel.svg" /></p>
+<p><img alt="Scheduler Highlevel" src="/plantuml/scheduler_highlevel.svg" /></p>
 <h2 id="performance">Performance</h2>
 <p>As of this writing, the scheduler is configured to utilize 256 threads for collecting.
 Because of the nature of collections (it's a lot of do wait), most of the time
@@ -206,7 +206,7 @@ <h3 id="probable">Probable</h3>
 shows one LC, it's likely we'd have a node in every LC that we had along with
 (potentially) additional nodes for Cloud Assets if that was the direction we wanted
 to go.</p>
-<p><img alt="Scheduler Proposed Future State" src="../../plantuml/scheduler_future_state.svg" /></p>
+<p><img alt="Scheduler Proposed Future State" src="/plantuml/scheduler_future_state.svg" /></p>
 <p>In a fully realized, microservices architecture the Scheduler would be obsolete.</p>
               
             </div>
diff --git a/docs/overview/index.html b/docs/overview/index.html
index 1487d84..d59e0f5 100644
--- a/docs/overview/index.html
+++ b/docs/overview/index.html
@@ -50,7 +50,7 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="..">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
diff --git a/docs/search.html b/docs/search.html
index 9021d57..82323e1 100644
--- a/docs/search.html
+++ b/docs/search.html
@@ -43,7 +43,7 @@
           
             <li class="toctree-l1">
 		
-    <span class="caption-text">Man o' War</span>
+    <a class="" href="./.">Man o' War</a>
 	    </li>
           
             <li class="toctree-l1">
diff --git a/docs/search/main.js b/docs/search/main.js
index 924a7a2..0a82ab5 100644
--- a/docs/search/main.js
+++ b/docs/search/main.js
@@ -9,8 +9,20 @@ function getSearchTermFromLocation() {
   }
 }
 
+function joinUrl (base, path) {
+  if (path.substring(0, 1) === "/") {
+    // path starts with `/`. Thus it is absolute.
+    return path;
+  }
+  if (base.substring(base.length-1) === "/") {
+    // base ends with `/`
+    return base + path;
+  }
+  return base + "/" + path;
+}
+
 function formatResult (location, title, summary) {
-  return '<article><h3><a href="' + base_url + '/' + location + '">'+ title + '</a></h3><p>' + summary +'</p></article>';
+  return '<article><h3><a href="' + joinUrl(base_url, location) + '">'+ title + '</a></h3><p>' + summary +'</p></article>';
 }
 
 function displayResults (results) {
@@ -32,7 +44,6 @@ function displayResults (results) {
 function doSearch () {
   var query = document.getElementById('mkdocs-search-query').value;
   if (query.length > 2) {
-    console.log('Searching with query: ' + query);
     if (!window.Worker) {
       displayResults(search(query));
     } else {
@@ -49,7 +60,6 @@ function initSearch () {
   if (search_input) {
     search_input.addEventListener("keyup", doSearch);
   }
-
   var term = getSearchTermFromLocation();
   if (term) {
     search_input.value = term;
@@ -58,7 +68,9 @@ function initSearch () {
 }
 
 function onWorkerMessage (e) {
-  if (e.data.results) {
+  if (e.data.allowSearch) {
+    initSearch();
+  } else if (e.data.results) {
     var results = e.data.results;
     displayResults(results);
   }
@@ -67,28 +79,18 @@ function onWorkerMessage (e) {
 if (!window.Worker) {
   console.log('Web Worker API not supported');
   // load index in main thread
-  $.getScript(base_url + "/search/worker.js").done(function () {
+  $.getScript(joinUrl(base_url, "search/worker.js")).done(function () {
     console.log('Loaded worker');
     init();
+    window.postMessage = function (msg) {
+      onWorkerMessage({data: msg});
+    };
   }).fail(function (jqxhr, settings, exception) {
     console.error('Could not load worker.js');
   });
 } else {
   // Wrap search in a web worker
-  var searchWorker = new Worker(base_url + "/search/worker.js");
+  var searchWorker = new Worker(joinUrl(base_url, "search/worker.js"));
   searchWorker.postMessage({init: true});
   searchWorker.onmessage = onWorkerMessage;
 }
-
-$(function() {
-  var search_input = document.getElementById('mkdocs-search-query');
-  if (search_input) {
-    search_input.addEventListener("keyup", doSearch);
-  }
-
-  var term = getSearchTermFromLocation();
-  if (term) {
-    search_input.value = term;
-    doSearch();
-  }
-});
diff --git a/docs/search/search_index.json b/docs/search/search_index.json
index f3beee7..19e4108 100644
--- a/docs/search/search_index.json
+++ b/docs/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Man o' War Introduction Man o' War has a goal of collecting more data, allowing historical lookbacks, and providing a more flexible auditing solution. Additionally it has a goal of having a flexible model for metadata storage so that future collection and analysis needs can be met. It was based of a project VDMS named called Jellyfish (it should be seen as an opensourced fork of the project). There are name references to it all over the place so keep that in mind.","title":"Man o' War"},{"location":"#man-o-war","text":"","title":"Man o' War"},{"location":"#introduction","text":"Man o' War has a goal of collecting more data, allowing historical lookbacks, and providing a more flexible auditing solution. Additionally it has a goal of having a flexible model for metadata storage so that future collection and analysis needs can be met. It was based of a project VDMS named called Jellyfish (it should be seen as an opensourced fork of the project). There are name references to it all over the place so keep that in mind.","title":"Introduction"},{"location":"authentication/","text":"Authentication Introduction Authentication in Jellyfish reflects my limited programming ability at the time of creation. Initially it had no concept of authorization and anyone who could authenticate the the api could do anything they wished. On the backend side, I used one ssh key to authenticate to the edge to do my collections.","title":"Authentication"},{"location":"authentication/#authentication","text":"","title":"Authentication"},{"location":"authentication/#introduction","text":"Authentication in Jellyfish reflects my limited programming ability at the time of creation. Initially it had no concept of authorization and anyone who could authenticate the the api could do anything they wished. On the backend side, I used one ssh key to authenticate to the edge to do my collections.","title":"Introduction"},{"location":"overview/","text":"Man o' War Introduction Man o' War has a goal of collecting more data, allowing historical lookbacks, and providing a more flexible auditing solution. Additionally it has a goal of having a flexible model for metadata storage so that future collection and analysis needs can be met. Man o' War Overview Man o' War is a follows the Unix philosophy of designing small modules to do a job well and then tie those modules together to do more complex things. At the moment Jellyfish is composed of 12 modules (with more being added as needed): Analyze BundleUSNs Collate Collector PullSwagger Schedule ScrapeUSN StorageAPI StorageJSONVerify Storage UI VerifyAudits Each module has a specific purpose and is deisgned to be able to run independently from each other (for ease of troubleshooting). They're brought together by either calling each other or by helper scripts (to be called by cron or init). There are several ideas for which module should come next. We always love pull requests so feel free to let us know about a few. Modules Collector The collector is designed to grab data back for one host. The first and current collector utilizes paramiko/ssh to log onto it's host, run a series of commands configured in collector.ini (and stored in SVN). If ran by hand you can utilize command line flags to test the data being given by a particular host. The collector will return a json or python dictionary that meets the standards specified in travis/artifacts/jellyfish_storage.json.schema . You can utilize the StorageJSONVerify module to confirm the goodness (or badness) of a particular set of data. Storage The storage module is designed to take json from a collector (that meets the travis/artifacts/jellyfish_storage.json.schema specification) and \"do the right thing\" for storage. For each collection it will query the database to see if there are changes. If there are, it will insert a new record with the proper timestamps. If there are not, it will update the existing record with the the current time (more specifically the time noted in the json); unless the time given is less than the time currently on disk (think of race conditions). Data is stored in the database as \"Vectors\" with the time being the magnitude of the vector the various data as the direction (See Diagram). The Database the storage module uses is a MariaDB 10.1 (or higher) database. It's connection details are configured in storage.ini . Additionally it's schema is stored in the setup/jellyfish2_db_schema.sql . Scheduler The scheduler is called by cron (at the moment) and it is fenced with a lockfile in /var/run/jellyfish (should be somewhat configurable). The scheduler utilizes a configurable amount of threads (configured in travis/artifacts/scheduler.ini as an example) to run a number of instances of the Collector Storage modules. It uses the server4.csv file located in the netinfo svn location to grab a list of servers it needs to check. The module will quit after a certain amount of time where it can't reach all the hosts. In this scenario it will return an item in the output json that looks like this: \"Timeout\": \"Timeout reached at 57627.98305392265 seconds with 29 items left on the queue.\", The Verbose module will output a status message to stdout for the duration of the run. Additionally a final status json will be outputted to stdout (and optionally to a seperate file) that contains the following pieces of information: global_fail_hosts - How many hosts scheduler failed to collect data from global_fail_hosts_list - A list of those hosts global_fail_prod - How many hosts scheduler failed to collect data from that were listed as \"production\" in their uber status. global_fail_prod_list - A list of those production hosts. global_success_hosts - How many hosts scheduler successfully collected data from. global_success_hosts_list - A list of those hosts. jobtime - How long, in seconds, scheduler ran for. threads - How many threads the system used Analyze See modules/analyze . Collate See modules/collate . Verify Audits Verify audits will verify either a single audit or a recursive dictionary of audits to see if they \"make sense.\" This module makes heavy use of the ast.literal_eval to parse and analyze a file. It's utilized by several modules to verify that an audit makes sense before it is analyzed. Pull Swagger PullSwagger is a little tool that will recurse through a direcotry of python files and pull the first fenced swagger definition out of that file. Then it utilizes a jinja template (currenltly located at openapi3/openapi3.yml.jinja in SVN) to build a Swagger definition file from it. Currently it's used for the jelly_api_2 directory to pull a file that get's displayed here. Storage JSON Verify Is a module that will check a particular JSON file against json schema file ( travis/jellyfish_storage.json.schema in SVN) to see if it's valid. In theory it can check any given json against any given json schema file. UI It's a flask app that can be controlled on the main box by a service command (jellyfish2-ui). Additionally there's an Apache forwarder configured to point back to this flask app and provide LDAP authentication.","title":"Man o' War"},{"location":"overview/#man-o-war","text":"","title":"Man o' War"},{"location":"overview/#introduction","text":"Man o' War has a goal of collecting more data, allowing historical lookbacks, and providing a more flexible auditing solution. Additionally it has a goal of having a flexible model for metadata storage so that future collection and analysis needs can be met.","title":"Introduction"},{"location":"overview/#man-o-war-overview","text":"Man o' War is a follows the Unix philosophy of designing small modules to do a job well and then tie those modules together to do more complex things. At the moment Jellyfish is composed of 12 modules (with more being added as needed): Analyze BundleUSNs Collate Collector PullSwagger Schedule ScrapeUSN StorageAPI StorageJSONVerify Storage UI VerifyAudits Each module has a specific purpose and is deisgned to be able to run independently from each other (for ease of troubleshooting). They're brought together by either calling each other or by helper scripts (to be called by cron or init). There are several ideas for which module should come next. We always love pull requests so feel free to let us know about a few.","title":"Man o' War Overview"},{"location":"overview/#modules","text":"","title":"Modules"},{"location":"overview/#collector","text":"The collector is designed to grab data back for one host. The first and current collector utilizes paramiko/ssh to log onto it's host, run a series of commands configured in collector.ini (and stored in SVN). If ran by hand you can utilize command line flags to test the data being given by a particular host. The collector will return a json or python dictionary that meets the standards specified in travis/artifacts/jellyfish_storage.json.schema . You can utilize the StorageJSONVerify module to confirm the goodness (or badness) of a particular set of data.","title":"Collector"},{"location":"overview/#storage","text":"The storage module is designed to take json from a collector (that meets the travis/artifacts/jellyfish_storage.json.schema specification) and \"do the right thing\" for storage. For each collection it will query the database to see if there are changes. If there are, it will insert a new record with the proper timestamps. If there are not, it will update the existing record with the the current time (more specifically the time noted in the json); unless the time given is less than the time currently on disk (think of race conditions). Data is stored in the database as \"Vectors\" with the time being the magnitude of the vector the various data as the direction (See Diagram). The Database the storage module uses is a MariaDB 10.1 (or higher) database. It's connection details are configured in storage.ini . Additionally it's schema is stored in the setup/jellyfish2_db_schema.sql .","title":"Storage"},{"location":"overview/#scheduler","text":"The scheduler is called by cron (at the moment) and it is fenced with a lockfile in /var/run/jellyfish (should be somewhat configurable). The scheduler utilizes a configurable amount of threads (configured in travis/artifacts/scheduler.ini as an example) to run a number of instances of the Collector Storage modules. It uses the server4.csv file located in the netinfo svn location to grab a list of servers it needs to check. The module will quit after a certain amount of time where it can't reach all the hosts. In this scenario it will return an item in the output json that looks like this: \"Timeout\": \"Timeout reached at 57627.98305392265 seconds with 29 items left on the queue.\", The Verbose module will output a status message to stdout for the duration of the run. Additionally a final status json will be outputted to stdout (and optionally to a seperate file) that contains the following pieces of information: global_fail_hosts - How many hosts scheduler failed to collect data from global_fail_hosts_list - A list of those hosts global_fail_prod - How many hosts scheduler failed to collect data from that were listed as \"production\" in their uber status. global_fail_prod_list - A list of those production hosts. global_success_hosts - How many hosts scheduler successfully collected data from. global_success_hosts_list - A list of those hosts. jobtime - How long, in seconds, scheduler ran for. threads - How many threads the system used","title":"Scheduler"},{"location":"overview/#analyze","text":"See modules/analyze .","title":"Analyze"},{"location":"overview/#collate","text":"See modules/collate .","title":"Collate"},{"location":"overview/#verify-audits","text":"Verify audits will verify either a single audit or a recursive dictionary of audits to see if they \"make sense.\" This module makes heavy use of the ast.literal_eval to parse and analyze a file. It's utilized by several modules to verify that an audit makes sense before it is analyzed.","title":"Verify Audits"},{"location":"overview/#pull-swagger","text":"PullSwagger is a little tool that will recurse through a direcotry of python files and pull the first fenced swagger definition out of that file. Then it utilizes a jinja template (currenltly located at openapi3/openapi3.yml.jinja in SVN) to build a Swagger definition file from it. Currently it's used for the jelly_api_2 directory to pull a file that get's displayed here.","title":"Pull Swagger"},{"location":"overview/#storage-json-verify","text":"Is a module that will check a particular JSON file against json schema file ( travis/jellyfish_storage.json.schema in SVN) to see if it's valid. In theory it can check any given json against any given json schema file.","title":"Storage JSON Verify"},{"location":"overview/#ui","text":"It's a flask app that can be controlled on the main box by a service command (jellyfish2-ui). Additionally there's an Apache forwarder configured to point back to this flask app and provide LDAP authentication.","title":"UI"},{"location":"modules/analyze/","text":"Analyze Introduction The analyze module runs independently. It will, using the audits as defined in audits.d directory, analyze the collections table to find out which hosts passed or failed the audits. Then it stores the results in the audits_by_host table. It assigns a unique audit result id, the id numbers for the host and audit, the timestamps associated with this result, which bucket this result is in the result optionally the result text. These audit results power the dashboard that is used by jellyfish and are the expected way to interact and process data within the Jellyfish system. Audits Audits are stored in the jellyfishaudits project. There you'll find some documentation on the content and structure of documentation that's available to you. Analyze Architecture The general idea of analyze is quite simple. The process : Grabs a lists of hosts Uses generic_large_compare to find out which hosts are in which bucket For Each Bucket: Uses either generic_large_compare or subtype_large_compare to compute passing or failing scores. Uses generic_large_analysis_store to store results in audits_by_host table in the database. Generic Large Analysis Store This module is a big part of the reason that autocommits were turned off. By batching up all the inserts/updates together we avoid an row locking race condition on the database. Future State This is a candidate for creating a lambda esque method of generating this data. Because of the nature of lambda you could simplify this code to check one host against one audit where every check will see which (or if) bucket it falls into and then flood your DB using lambda execution limits to make sure you dont' hammer your database.","title":"Analyze"},{"location":"modules/analyze/#analyze","text":"","title":"Analyze"},{"location":"modules/analyze/#introduction","text":"The analyze module runs independently. It will, using the audits as defined in audits.d directory, analyze the collections table to find out which hosts passed or failed the audits. Then it stores the results in the audits_by_host table. It assigns a unique audit result id, the id numbers for the host and audit, the timestamps associated with this result, which bucket this result is in the result optionally the result text. These audit results power the dashboard that is used by jellyfish and are the expected way to interact and process data within the Jellyfish system.","title":"Introduction"},{"location":"modules/analyze/#audits","text":"Audits are stored in the jellyfishaudits project. There you'll find some documentation on the content and structure of documentation that's available to you.","title":"Audits"},{"location":"modules/analyze/#analyze-architecture","text":"The general idea of analyze is quite simple. The process : Grabs a lists of hosts Uses generic_large_compare to find out which hosts are in which bucket For Each Bucket: Uses either generic_large_compare or subtype_large_compare to compute passing or failing scores. Uses generic_large_analysis_store to store results in audits_by_host table in the database.","title":"Analyze Architecture"},{"location":"modules/analyze/#generic-large-analysis-store","text":"This module is a big part of the reason that autocommits were turned off. By batching up all the inserts/updates together we avoid an row locking race condition on the database.","title":"Generic Large Analysis Store"},{"location":"modules/analyze/#future-state","text":"This is a candidate for creating a lambda esque method of generating this data. Because of the nature of lambda you could simplify this code to check one host against one audit where every check will see which (or if) bucket it falls into and then flood your DB using lambda execution limits to make sure you dont' hammer your database.","title":"Future State"},{"location":"modules/canonical_cve/","text":"Canonical CVE This is actually a simple module. Give this command a CVE number and it will query Ubuntu's CVE site to get any associated package data for the CVE in question. The following is an example with CVE-2018-8885 . ./canonical_cve.py CVE-2018-8885 | jq '.' { references : [ https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-8885 , https://launchpad.net/bugs/1753772 , https://usn.ubuntu.com/usn/usn-3607-1 ], title : CVE-2018-8885 , priority : Medium , packages : { screen-resolution-extra : { package_name : screen-resolution-extra , package_upstream_status : needs-triage , releases : { precise : { release : precise , status : dne , version : false }, trusty : { release : trusty , status : released , version : 0.17.1.1~14.04.1 }, xenial : { release : xenial , status : released , version : 0.17.1.1~16.04.1 }, artful : { release : artful , status : released , version : 0.17.1.1 }, bionic : { release : bionic , status : needed , version : false } } } } } Usage This is used in a few other modules (like bass in jellyfishaudits) to get information about CVEs. And is used in sherlockfish to do semi-automatic, on demand audits of information. Future This is a webscraper. If ever Ubuntu updates it's formatting we'll need to update our scraping. It's probably a good idea to instead of scraping the website go straight to the source for our future needs.","title":"Canonical CVE"},{"location":"modules/canonical_cve/#canonical-cve","text":"This is actually a simple module. Give this command a CVE number and it will query Ubuntu's CVE site to get any associated package data for the CVE in question. The following is an example with CVE-2018-8885 . ./canonical_cve.py CVE-2018-8885 | jq '.' { references : [ https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-8885 , https://launchpad.net/bugs/1753772 , https://usn.ubuntu.com/usn/usn-3607-1 ], title : CVE-2018-8885 , priority : Medium , packages : { screen-resolution-extra : { package_name : screen-resolution-extra , package_upstream_status : needs-triage , releases : { precise : { release : precise , status : dne , version : false }, trusty : { release : trusty , status : released , version : 0.17.1.1~14.04.1 }, xenial : { release : xenial , status : released , version : 0.17.1.1~16.04.1 }, artful : { release : artful , status : released , version : 0.17.1.1 }, bionic : { release : bionic , status : needed , version : false } } } } }","title":"Canonical CVE"},{"location":"modules/canonical_cve/#usage","text":"This is used in a few other modules (like bass in jellyfishaudits) to get information about CVEs. And is used in sherlockfish to do semi-automatic, on demand audits of information.","title":"Usage"},{"location":"modules/canonical_cve/#future","text":"This is a webscraper. If ever Ubuntu updates it's formatting we'll need to update our scraping. It's probably a good idea to instead of scraping the website go straight to the source for our future needs.","title":"Future"},{"location":"modules/collate/","text":"Collate Introduction We used to collate in-line in the Analyze module. However that proved to be quite costly. So we moved to a system where we analyze and then we collate our data seperately. This module does that. It figures passed, failed and exempt audits on a per-audit, per-pop and per-srvtype basis. Arch The idea here is simple. It will cycle through the various by factors (pop, srvtype audit/acoll). It counts up the results in audits_by_host for each of these factors and then update the related numbers in audits_by_factor . Future State This probably couldn't be moved to a lambda (unless the future database is quite quick) but it could be an excellent candidate for Amazon Glue as it's in effect a simplistic ETL job.","title":"Collate"},{"location":"modules/collate/#collate","text":"","title":"Collate"},{"location":"modules/collate/#introduction","text":"We used to collate in-line in the Analyze module. However that proved to be quite costly. So we moved to a system where we analyze and then we collate our data seperately. This module does that. It figures passed, failed and exempt audits on a per-audit, per-pop and per-srvtype basis.","title":"Introduction"},{"location":"modules/collate/#arch","text":"The idea here is simple. It will cycle through the various by factors (pop, srvtype audit/acoll). It counts up the results in audits_by_host for each of these factors and then update the related numbers in audits_by_factor .","title":"Arch"},{"location":"modules/collate/#future-state","text":"This probably couldn't be moved to a lambda (unless the future database is quite quick) but it could be an excellent candidate for Amazon Glue as it's in effect a simplistic ETL job.","title":"Future State"},{"location":"modules/collection_archive/","text":"Collection Archive The collections table is big and while it is (in my opinion) well indexed even well indexed tables get too big eventually. This implments archiving of older collection data. Data that has been obsolete for 7 days get's moved from the collection table to the collection_archive table. This get's ran daily as a part of the run_schedule.sh job sequence. Future It's likely that this could be made obsolete in the future with better sharding. Additionally this could probably be turned into a serverless function if ever the decision was made to migrate to the cloud.","title":"Collection Archive"},{"location":"modules/collection_archive/#collection-archive","text":"The collections table is big and while it is (in my opinion) well indexed even well indexed tables get too big eventually. This implments archiving of older collection data. Data that has been obsolete for 7 days get's moved from the collection table to the collection_archive table. This get's ran daily as a part of the run_schedule.sh job sequence.","title":"Collection Archive"},{"location":"modules/collection_archive/#future","text":"It's likely that this could be made obsolete in the future with better sharding. Additionally this could probably be turned into a serverless function if ever the decision was made to migrate to the cloud.","title":"Future"},{"location":"modules/collector/","text":"Collector By itself collector.py is actually a simple module. It simply parses some configs to tell itself what it out to do. Then runs and parses those commands on individual hosts. It's main usage is to get called by schedule2.py over and over again to do the collections. Collection Configuration Collection configuration is by default stored in svn in the collector.ini file. Here is an example of a collection : [packages] ; Remote Collection Command multi=TRUE collection=dpkg --list | grep -E '^ii' | awk '{print $2 \\011 $3}' Collections can be either single dimentional or multi-dimensional. Single dimentional lines will get stored as just one \"default\" collection subtype. Multidimentional ones will get stored with each line being treated as a subtype value combo where whitespace seperates the two items. Here are the top few results of what this command produces. Note how each package will end up being it's own subtype. : dpkg --list | grep -E '^ii' | awk '{print $2 \\011 $3}' | head accountsservice 0.6.42-0ubuntu3.1 acl 2.2.52-3build1 acpi-support 0.142 acpid 1:2.0.28-1ubuntu1 adduser 3.113+nmu3ubuntu5 adium-theme-ubuntu 0.3.4-0ubuntu4 adobe-flash-properties-gtk 1:20180313.1-0ubuntu0.17.10.1 adobe-flashplugin 1:20180313.1-0ubuntu0.17.10.1 adwaita-icon-theme 3.26.0-0ubuntu2 aisleriot 1:3.22.2-0ubuntu1 Interpolation is a thing. When the config is parsed any % character is going to be specially viewed. If you need a % character (see example below) you'll need to write a %% . You'll find it useful with stat and date commands. Here is an example of a collection that uses this: [ssh-host-key-age] ; Grabs a hash of the SSH host key multi=TRUE collection=stat -c %%n %%Y /etc/ssh/*.pub | tr '[/.]' '_' SSH Python's Paramiko module is used to log into each host and the jellyfish ssh key that get's installed in salt is generally used to log in. Currently that key is managed by hand on the jellyfish machine and it's public half is delivered via salt. Example Below is an example of a collection being ran manually and the data it produces (truncated to protect the innocent and the scrollwheels). Note that because of the uniqueness of bast servers a good chunk of collections don't work out of the box. { collection_data : { boottime : { default : Tue Mar 20 00:00:00 UTC 2018 }, cpu-info : { Architecture: : x86_64 , BogoMIPS: : 4000.00 , Byte : Order: Little Endian , CPU : MHz: 2000.000 , CPU(s): : 8 , Core(s) : per socket: 4 , Hypervisor : vendor: VMware , L1d : cache: 32K , L1i : cache: 32K , L2 : cache: 256K , L3 : cache: 15360K , Model: : 45 , NUMA : node0 CPU(s): 0-7 , On-line : CPU(s) list: 0-7 , Socket(s): : 2 , Stepping: : 7 , Thread(s) : per core: 1 , Vendor : ID: GenuineIntel , Virtualization : type: full }, host_host : { HOSTNAME : host1.host , POP : N/A , SRVTYPE : N/A , STATUS : N/A , UBERID : N/A }, pci-info : { 00:00.0 : Host bridge: Intel Corporation 440BX/ZX/DX - 82443BX/ZX/DX Host bridge (rev 01) , 00:01.0 : PCI bridge: Intel Corporation 440BX/ZX/DX - 82443BX/ZX/DX AGP bridge (rev 01) , 00:07.0 : ISA bridge: Intel Corporation 82371AB/EB/MB PIIX4 ISA (rev 08) , 00:07.1 : IDE interface: Intel Corporation 82371AB/EB/MB PIIX4 IDE (rev 01) , 00:07.3 : Bridge: Intel Corporation 82371AB/EB/MB PIIX4 ACPI (rev 08) , 00:07.7 : System peripheral: VMware Virtual Machine Communication Interface (rev 10) , 00:0f.0 : VGA compatible controller: VMware SVGA II Adapter , 00:11.0 : PCI bridge: VMware PCI bridge (rev 02) , 00:15.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 03:00.0 : Ethernet controller: VMware VMXNET3 Ethernet Controller (rev 01) }, reboot-needed : { default : NO-REBOOT-REQUIRED }, release : { default : trusty }, repositories-hash : { default : 1fe60f07c344e2a4e9459be05aed9c12+8 }, rkernel : { default : 4.4.0-103-generic } }, collection_hostname : host1.host , collection_status : SSH SUCCESS , collection_timestamp : 1522187510, connection_string : chalbersma@1.1.1.1 , ip_intel : [ { ip : 1.1.1.1 , iptype : host4 } ], ipv6_val_error : Found IPV6 Address would not Validateillegal IP address string passed to inet_pton , pop : N/A , srvtype : N/A , status : N/A , uber_id : N/A }","title":"Collector"},{"location":"modules/collector/#collector","text":"By itself collector.py is actually a simple module. It simply parses some configs to tell itself what it out to do. Then runs and parses those commands on individual hosts. It's main usage is to get called by schedule2.py over and over again to do the collections.","title":"Collector"},{"location":"modules/collector/#collection-configuration","text":"Collection configuration is by default stored in svn in the collector.ini file. Here is an example of a collection : [packages] ; Remote Collection Command multi=TRUE collection=dpkg --list | grep -E '^ii' | awk '{print $2 \\011 $3}' Collections can be either single dimentional or multi-dimensional. Single dimentional lines will get stored as just one \"default\" collection subtype. Multidimentional ones will get stored with each line being treated as a subtype value combo where whitespace seperates the two items. Here are the top few results of what this command produces. Note how each package will end up being it's own subtype. : dpkg --list | grep -E '^ii' | awk '{print $2 \\011 $3}' | head accountsservice 0.6.42-0ubuntu3.1 acl 2.2.52-3build1 acpi-support 0.142 acpid 1:2.0.28-1ubuntu1 adduser 3.113+nmu3ubuntu5 adium-theme-ubuntu 0.3.4-0ubuntu4 adobe-flash-properties-gtk 1:20180313.1-0ubuntu0.17.10.1 adobe-flashplugin 1:20180313.1-0ubuntu0.17.10.1 adwaita-icon-theme 3.26.0-0ubuntu2 aisleriot 1:3.22.2-0ubuntu1 Interpolation is a thing. When the config is parsed any % character is going to be specially viewed. If you need a % character (see example below) you'll need to write a %% . You'll find it useful with stat and date commands. Here is an example of a collection that uses this: [ssh-host-key-age] ; Grabs a hash of the SSH host key multi=TRUE collection=stat -c %%n %%Y /etc/ssh/*.pub | tr '[/.]' '_'","title":"Collection Configuration"},{"location":"modules/collector/#ssh","text":"Python's Paramiko module is used to log into each host and the jellyfish ssh key that get's installed in salt is generally used to log in. Currently that key is managed by hand on the jellyfish machine and it's public half is delivered via salt.","title":"SSH"},{"location":"modules/collector/#example","text":"Below is an example of a collection being ran manually and the data it produces (truncated to protect the innocent and the scrollwheels). Note that because of the uniqueness of bast servers a good chunk of collections don't work out of the box. { collection_data : { boottime : { default : Tue Mar 20 00:00:00 UTC 2018 }, cpu-info : { Architecture: : x86_64 , BogoMIPS: : 4000.00 , Byte : Order: Little Endian , CPU : MHz: 2000.000 , CPU(s): : 8 , Core(s) : per socket: 4 , Hypervisor : vendor: VMware , L1d : cache: 32K , L1i : cache: 32K , L2 : cache: 256K , L3 : cache: 15360K , Model: : 45 , NUMA : node0 CPU(s): 0-7 , On-line : CPU(s) list: 0-7 , Socket(s): : 2 , Stepping: : 7 , Thread(s) : per core: 1 , Vendor : ID: GenuineIntel , Virtualization : type: full }, host_host : { HOSTNAME : host1.host , POP : N/A , SRVTYPE : N/A , STATUS : N/A , UBERID : N/A }, pci-info : { 00:00.0 : Host bridge: Intel Corporation 440BX/ZX/DX - 82443BX/ZX/DX Host bridge (rev 01) , 00:01.0 : PCI bridge: Intel Corporation 440BX/ZX/DX - 82443BX/ZX/DX AGP bridge (rev 01) , 00:07.0 : ISA bridge: Intel Corporation 82371AB/EB/MB PIIX4 ISA (rev 08) , 00:07.1 : IDE interface: Intel Corporation 82371AB/EB/MB PIIX4 IDE (rev 01) , 00:07.3 : Bridge: Intel Corporation 82371AB/EB/MB PIIX4 ACPI (rev 08) , 00:07.7 : System peripheral: VMware Virtual Machine Communication Interface (rev 10) , 00:0f.0 : VGA compatible controller: VMware SVGA II Adapter , 00:11.0 : PCI bridge: VMware PCI bridge (rev 02) , 00:15.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 03:00.0 : Ethernet controller: VMware VMXNET3 Ethernet Controller (rev 01) }, reboot-needed : { default : NO-REBOOT-REQUIRED }, release : { default : trusty }, repositories-hash : { default : 1fe60f07c344e2a4e9459be05aed9c12+8 }, rkernel : { default : 4.4.0-103-generic } }, collection_hostname : host1.host , collection_status : SSH SUCCESS , collection_timestamp : 1522187510, connection_string : chalbersma@1.1.1.1 , ip_intel : [ { ip : 1.1.1.1 , iptype : host4 } ], ipv6_val_error : Found IPV6 Address would not Validateillegal IP address string passed to inet_pton , pop : N/A , srvtype : N/A , status : N/A , uber_id : N/A }","title":"Example"},{"location":"modules/scheduler/","text":"Scheduler Introduction The scheduler is a module that essentially interacts with three other modules, collector , storage and sapicheck . It ties these together and in a big ole loop enumerates the systems in Uber and attempts to profile them. Performance As of this writing, the scheduler is configured to utilize 256 threads for collecting. Because of the nature of collections (it's a lot of do wait), most of the time you spend waiting for processes to return on the target hosts. The 256 threads limitiation is likely a database limitation. Because of these limitations the scheduler is limited to run for 8 hours a day. It prioritizes systems in production and ends up behind schedule quite a bit. The orignal plan was to migrate this into EBY but before we could do that SecEng was Oathafied. So the future of the product will be decided by whomever takes it over. Input/Output The scheduler is called by cron (at the moment) and it is fenced with a lockfile in /var/run/jellyfish . The scheduler utilizes a configurable amount of threads (configured in scheduler.ini in SVN) to run a number of instances of the Collector Storage modules. It uses the server4.csv file located in the netinfo svn location to grab a list of servers it needs to check. The module will quit after a certain amount of time where it can't reach all the hosts. In this scenario it will return an item in the output json that looks like this: \"Timeout\": \"Timeout reached at 57627.98305392265 seconds with 29 items left on the queue.\", The Verbose module will output a status message to stdout for the duration of the run. Additionally a final status json will be outputted to stdout (and optionally to a seperate file) that contains the following pieces of information: global_fail_hosts - How many hosts scheduler failed to collect data from global_fail_hosts_list - A list of those hosts global_fail_prod - How many hosts scheduler failed that were listed as \"production\". global_fail_prod_list - A list of those production hosts. global_success_hosts - How many hosts scheduler successfully collected data from. global_success_hosts_list - A list of those hosts. jobtime - How long, in seconds, scheduler ran for. threads - How many threads the system used Gap Fix Gap fix is a related module to Scheduler. It's designed to find systems that are in prod in uber but don't have a recent result. It works almost the same as scheduler but has a lighter load so that it doesn't interrupt normal operations. Most importantly it doesn't trigger the lock file that keeps analysis from running. Future State Probable Ideally this would in the future be replaced with stingcell, an agent that runs on hosts and reports information back via the storage api. Scheduler would likely have a funcationality similar to gap_fix and only stand to attempt to login to systems that hadn't checked in via the api for a period of time. It would be likely that a box in EC IR500 Command and Control environments would exist with API keys that can log into the missing hosts (See Diagram). And it, like stingcell would utlize the API to make it's storage calls. While the document only shows one LC, it's likely we'd have a node in every LC that we had along with (potentially) additional nodes for Cloud Assets if that was the direction we wanted to go. In a fully realized, microservices architecture the Scheduler would be obsolete.","title":"Scheduler"},{"location":"modules/scheduler/#scheduler","text":"","title":"Scheduler"},{"location":"modules/scheduler/#introduction","text":"The scheduler is a module that essentially interacts with three other modules, collector , storage and sapicheck . It ties these together and in a big ole loop enumerates the systems in Uber and attempts to profile them.","title":"Introduction"},{"location":"modules/scheduler/#performance","text":"As of this writing, the scheduler is configured to utilize 256 threads for collecting. Because of the nature of collections (it's a lot of do wait), most of the time you spend waiting for processes to return on the target hosts. The 256 threads limitiation is likely a database limitation. Because of these limitations the scheduler is limited to run for 8 hours a day. It prioritizes systems in production and ends up behind schedule quite a bit. The orignal plan was to migrate this into EBY but before we could do that SecEng was Oathafied. So the future of the product will be decided by whomever takes it over.","title":"Performance"},{"location":"modules/scheduler/#inputoutput","text":"The scheduler is called by cron (at the moment) and it is fenced with a lockfile in /var/run/jellyfish . The scheduler utilizes a configurable amount of threads (configured in scheduler.ini in SVN) to run a number of instances of the Collector Storage modules. It uses the server4.csv file located in the netinfo svn location to grab a list of servers it needs to check. The module will quit after a certain amount of time where it can't reach all the hosts. In this scenario it will return an item in the output json that looks like this: \"Timeout\": \"Timeout reached at 57627.98305392265 seconds with 29 items left on the queue.\", The Verbose module will output a status message to stdout for the duration of the run. Additionally a final status json will be outputted to stdout (and optionally to a seperate file) that contains the following pieces of information: global_fail_hosts - How many hosts scheduler failed to collect data from global_fail_hosts_list - A list of those hosts global_fail_prod - How many hosts scheduler failed that were listed as \"production\". global_fail_prod_list - A list of those production hosts. global_success_hosts - How many hosts scheduler successfully collected data from. global_success_hosts_list - A list of those hosts. jobtime - How long, in seconds, scheduler ran for. threads - How many threads the system used","title":"Input/Output"},{"location":"modules/scheduler/#gap-fix","text":"Gap fix is a related module to Scheduler. It's designed to find systems that are in prod in uber but don't have a recent result. It works almost the same as scheduler but has a lighter load so that it doesn't interrupt normal operations. Most importantly it doesn't trigger the lock file that keeps analysis from running.","title":"Gap Fix"},{"location":"modules/scheduler/#future-state","text":"","title":"Future State"},{"location":"modules/scheduler/#probable","text":"Ideally this would in the future be replaced with stingcell, an agent that runs on hosts and reports information back via the storage api. Scheduler would likely have a funcationality similar to gap_fix and only stand to attempt to login to systems that hadn't checked in via the api for a period of time. It would be likely that a box in EC IR500 Command and Control environments would exist with API keys that can log into the missing hosts (See Diagram). And it, like stingcell would utlize the API to make it's storage calls. While the document only shows one LC, it's likely we'd have a node in every LC that we had along with (potentially) additional nodes for Cloud Assets if that was the direction we wanted to go. In a fully realized, microservices architecture the Scheduler would be obsolete.","title":"Probable"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Man o' War Introduction Man o' War has a goal of collecting more data, allowing historical lookbacks, and providing a more flexible auditing solution. Additionally it has a goal of having a flexible model for metadata storage so that future collection and analysis needs can be met. It was based of a project VDMS named called Jellyfish (it should be seen as an opensourced fork of the project). There are name references to it all over the place so keep that in mind. Source Code","title":"Man o' War"},{"location":"#man-o-war","text":"","title":"Man o' War"},{"location":"#introduction","text":"Man o' War has a goal of collecting more data, allowing historical lookbacks, and providing a more flexible auditing solution. Additionally it has a goal of having a flexible model for metadata storage so that future collection and analysis needs can be met. It was based of a project VDMS named called Jellyfish (it should be seen as an opensourced fork of the project). There are name references to it all over the place so keep that in mind. Source Code","title":"Introduction"},{"location":"authentication/","text":"Authentication Introduction Authentication in Jellyfish reflects my limited programming ability at the time of creation. Initially it had no concept of authorization and anyone who could authenticate the the api could do anything they wished. On the backend side, I used one ssh key to authenticate to the edge to do my collections.","title":"Authentication"},{"location":"authentication/#authentication","text":"","title":"Authentication"},{"location":"authentication/#introduction","text":"Authentication in Jellyfish reflects my limited programming ability at the time of creation. Initially it had no concept of authorization and anyone who could authenticate the the api could do anything they wished. On the backend side, I used one ssh key to authenticate to the edge to do my collections.","title":"Introduction"},{"location":"overview/","text":"Man o' War Introduction Man o' War has a goal of collecting more data, allowing historical lookbacks, and providing a more flexible auditing solution. Additionally it has a goal of having a flexible model for metadata storage so that future collection and analysis needs can be met. Man o' War Overview Man o' War is a follows the Unix philosophy of designing small modules to do a job well and then tie those modules together to do more complex things. At the moment Jellyfish is composed of 12 modules (with more being added as needed): Analyze BundleUSNs Collate Collector PullSwagger Schedule ScrapeUSN StorageAPI StorageJSONVerify Storage UI VerifyAudits Each module has a specific purpose and is deisgned to be able to run independently from each other (for ease of troubleshooting). They're brought together by either calling each other or by helper scripts (to be called by cron or init). There are several ideas for which module should come next. We always love pull requests so feel free to let us know about a few. Modules Collector The collector is designed to grab data back for one host. The first and current collector utilizes paramiko/ssh to log onto it's host, run a series of commands configured in collector.ini (and stored in SVN). If ran by hand you can utilize command line flags to test the data being given by a particular host. The collector will return a json or python dictionary that meets the standards specified in travis/artifacts/jellyfish_storage.json.schema . You can utilize the StorageJSONVerify module to confirm the goodness (or badness) of a particular set of data. Storage The storage module is designed to take json from a collector (that meets the travis/artifacts/jellyfish_storage.json.schema specification) and \"do the right thing\" for storage. For each collection it will query the database to see if there are changes. If there are, it will insert a new record with the proper timestamps. If there are not, it will update the existing record with the the current time (more specifically the time noted in the json); unless the time given is less than the time currently on disk (think of race conditions). Data is stored in the database as \"Vectors\" with the time being the magnitude of the vector the various data as the direction (See Diagram). The Database the storage module uses is a MariaDB 10.1 (or higher) database. It's connection details are configured in storage.ini . Additionally it's schema is stored in the setup/jellyfish2_db_schema.sql . Scheduler The scheduler is called by cron (at the moment) and it is fenced with a lockfile in /var/run/jellyfish (should be somewhat configurable). The scheduler utilizes a configurable amount of threads (configured in travis/artifacts/scheduler.ini as an example) to run a number of instances of the Collector Storage modules. It uses the server4.csv file located in the netinfo svn location to grab a list of servers it needs to check. The module will quit after a certain amount of time where it can't reach all the hosts. In this scenario it will return an item in the output json that looks like this: \"Timeout\": \"Timeout reached at 57627.98305392265 seconds with 29 items left on the queue.\", The Verbose module will output a status message to stdout for the duration of the run. Additionally a final status json will be outputted to stdout (and optionally to a seperate file) that contains the following pieces of information: global_fail_hosts - How many hosts scheduler failed to collect data from global_fail_hosts_list - A list of those hosts global_fail_prod - How many hosts scheduler failed to collect data from that were listed as \"production\" in their uber status. global_fail_prod_list - A list of those production hosts. global_success_hosts - How many hosts scheduler successfully collected data from. global_success_hosts_list - A list of those hosts. jobtime - How long, in seconds, scheduler ran for. threads - How many threads the system used Analyze See modules/analyze . Collate See modules/collate . Verify Audits Verify audits will verify either a single audit or a recursive dictionary of audits to see if they \"make sense.\" This module makes heavy use of the ast.literal_eval to parse and analyze a file. It's utilized by several modules to verify that an audit makes sense before it is analyzed. Pull Swagger PullSwagger is a little tool that will recurse through a direcotry of python files and pull the first fenced swagger definition out of that file. Then it utilizes a jinja template (currenltly located at openapi3/openapi3.yml.jinja in SVN) to build a Swagger definition file from it. Currently it's used for the jelly_api_2 directory to pull a file that get's displayed here. Storage JSON Verify Is a module that will check a particular JSON file against json schema file ( travis/jellyfish_storage.json.schema in SVN) to see if it's valid. In theory it can check any given json against any given json schema file. UI It's a flask app that can be controlled on the main box by a service command (jellyfish2-ui). Additionally there's an Apache forwarder configured to point back to this flask app and provide LDAP authentication.","title":"Man o' War"},{"location":"overview/#man-o-war","text":"","title":"Man o' War"},{"location":"overview/#introduction","text":"Man o' War has a goal of collecting more data, allowing historical lookbacks, and providing a more flexible auditing solution. Additionally it has a goal of having a flexible model for metadata storage so that future collection and analysis needs can be met.","title":"Introduction"},{"location":"overview/#man-o-war-overview","text":"Man o' War is a follows the Unix philosophy of designing small modules to do a job well and then tie those modules together to do more complex things. At the moment Jellyfish is composed of 12 modules (with more being added as needed): Analyze BundleUSNs Collate Collector PullSwagger Schedule ScrapeUSN StorageAPI StorageJSONVerify Storage UI VerifyAudits Each module has a specific purpose and is deisgned to be able to run independently from each other (for ease of troubleshooting). They're brought together by either calling each other or by helper scripts (to be called by cron or init). There are several ideas for which module should come next. We always love pull requests so feel free to let us know about a few.","title":"Man o' War Overview"},{"location":"overview/#modules","text":"","title":"Modules"},{"location":"overview/#collector","text":"The collector is designed to grab data back for one host. The first and current collector utilizes paramiko/ssh to log onto it's host, run a series of commands configured in collector.ini (and stored in SVN). If ran by hand you can utilize command line flags to test the data being given by a particular host. The collector will return a json or python dictionary that meets the standards specified in travis/artifacts/jellyfish_storage.json.schema . You can utilize the StorageJSONVerify module to confirm the goodness (or badness) of a particular set of data.","title":"Collector"},{"location":"overview/#storage","text":"The storage module is designed to take json from a collector (that meets the travis/artifacts/jellyfish_storage.json.schema specification) and \"do the right thing\" for storage. For each collection it will query the database to see if there are changes. If there are, it will insert a new record with the proper timestamps. If there are not, it will update the existing record with the the current time (more specifically the time noted in the json); unless the time given is less than the time currently on disk (think of race conditions). Data is stored in the database as \"Vectors\" with the time being the magnitude of the vector the various data as the direction (See Diagram). The Database the storage module uses is a MariaDB 10.1 (or higher) database. It's connection details are configured in storage.ini . Additionally it's schema is stored in the setup/jellyfish2_db_schema.sql .","title":"Storage"},{"location":"overview/#scheduler","text":"The scheduler is called by cron (at the moment) and it is fenced with a lockfile in /var/run/jellyfish (should be somewhat configurable). The scheduler utilizes a configurable amount of threads (configured in travis/artifacts/scheduler.ini as an example) to run a number of instances of the Collector Storage modules. It uses the server4.csv file located in the netinfo svn location to grab a list of servers it needs to check. The module will quit after a certain amount of time where it can't reach all the hosts. In this scenario it will return an item in the output json that looks like this: \"Timeout\": \"Timeout reached at 57627.98305392265 seconds with 29 items left on the queue.\", The Verbose module will output a status message to stdout for the duration of the run. Additionally a final status json will be outputted to stdout (and optionally to a seperate file) that contains the following pieces of information: global_fail_hosts - How many hosts scheduler failed to collect data from global_fail_hosts_list - A list of those hosts global_fail_prod - How many hosts scheduler failed to collect data from that were listed as \"production\" in their uber status. global_fail_prod_list - A list of those production hosts. global_success_hosts - How many hosts scheduler successfully collected data from. global_success_hosts_list - A list of those hosts. jobtime - How long, in seconds, scheduler ran for. threads - How many threads the system used","title":"Scheduler"},{"location":"overview/#analyze","text":"See modules/analyze .","title":"Analyze"},{"location":"overview/#collate","text":"See modules/collate .","title":"Collate"},{"location":"overview/#verify-audits","text":"Verify audits will verify either a single audit or a recursive dictionary of audits to see if they \"make sense.\" This module makes heavy use of the ast.literal_eval to parse and analyze a file. It's utilized by several modules to verify that an audit makes sense before it is analyzed.","title":"Verify Audits"},{"location":"overview/#pull-swagger","text":"PullSwagger is a little tool that will recurse through a direcotry of python files and pull the first fenced swagger definition out of that file. Then it utilizes a jinja template (currenltly located at openapi3/openapi3.yml.jinja in SVN) to build a Swagger definition file from it. Currently it's used for the jelly_api_2 directory to pull a file that get's displayed here.","title":"Pull Swagger"},{"location":"overview/#storage-json-verify","text":"Is a module that will check a particular JSON file against json schema file ( travis/jellyfish_storage.json.schema in SVN) to see if it's valid. In theory it can check any given json against any given json schema file.","title":"Storage JSON Verify"},{"location":"overview/#ui","text":"It's a flask app that can be controlled on the main box by a service command (jellyfish2-ui). Additionally there's an Apache forwarder configured to point back to this flask app and provide LDAP authentication.","title":"UI"},{"location":"modules/analyze/","text":"Analyze Introduction The analyze module runs independently. It will, using the audits as defined in audits.d directory, analyze the collections table to find out which hosts passed or failed the audits. Then it stores the results in the audits_by_host table. It assigns a unique audit result id, the id numbers for the host and audit, the timestamps associated with this result, which bucket this result is in the result optionally the result text. These audit results power the dashboard that is used by jellyfish and are the expected way to interact and process data within the Jellyfish system. Audits Audits are stored in the jellyfishaudits project. There you'll find some documentation on the content and structure of documentation that's available to you. Analyze Architecture The general idea of analyze is quite simple. The process : Grabs a lists of hosts Uses generic_large_compare to find out which hosts are in which bucket For Each Bucket: Uses either generic_large_compare or subtype_large_compare to compute passing or failing scores. Uses generic_large_analysis_store to store results in audits_by_host table in the database. Generic Large Analysis Store This module is a big part of the reason that autocommits were turned off. By batching up all the inserts/updates together we avoid an row locking race condition on the database. Future State This is a candidate for creating a lambda esque method of generating this data. Because of the nature of lambda you could simplify this code to check one host against one audit where every check will see which (or if) bucket it falls into and then flood your DB using lambda execution limits to make sure you dont' hammer your database.","title":"Analyze"},{"location":"modules/analyze/#analyze","text":"","title":"Analyze"},{"location":"modules/analyze/#introduction","text":"The analyze module runs independently. It will, using the audits as defined in audits.d directory, analyze the collections table to find out which hosts passed or failed the audits. Then it stores the results in the audits_by_host table. It assigns a unique audit result id, the id numbers for the host and audit, the timestamps associated with this result, which bucket this result is in the result optionally the result text. These audit results power the dashboard that is used by jellyfish and are the expected way to interact and process data within the Jellyfish system.","title":"Introduction"},{"location":"modules/analyze/#audits","text":"Audits are stored in the jellyfishaudits project. There you'll find some documentation on the content and structure of documentation that's available to you.","title":"Audits"},{"location":"modules/analyze/#analyze-architecture","text":"The general idea of analyze is quite simple. The process : Grabs a lists of hosts Uses generic_large_compare to find out which hosts are in which bucket For Each Bucket: Uses either generic_large_compare or subtype_large_compare to compute passing or failing scores. Uses generic_large_analysis_store to store results in audits_by_host table in the database.","title":"Analyze Architecture"},{"location":"modules/analyze/#generic-large-analysis-store","text":"This module is a big part of the reason that autocommits were turned off. By batching up all the inserts/updates together we avoid an row locking race condition on the database.","title":"Generic Large Analysis Store"},{"location":"modules/analyze/#future-state","text":"This is a candidate for creating a lambda esque method of generating this data. Because of the nature of lambda you could simplify this code to check one host against one audit where every check will see which (or if) bucket it falls into and then flood your DB using lambda execution limits to make sure you dont' hammer your database.","title":"Future State"},{"location":"modules/canonical_cve/","text":"Canonical CVE This is actually a simple module. Give this command a CVE number and it will query Ubuntu's CVE site to get any associated package data for the CVE in question. The following is an example with CVE-2018-8885 . ./canonical_cve.py CVE-2018-8885 | jq '.' { references : [ https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-8885 , https://launchpad.net/bugs/1753772 , https://usn.ubuntu.com/usn/usn-3607-1 ], title : CVE-2018-8885 , priority : Medium , packages : { screen-resolution-extra : { package_name : screen-resolution-extra , package_upstream_status : needs-triage , releases : { precise : { release : precise , status : dne , version : false }, trusty : { release : trusty , status : released , version : 0.17.1.1~14.04.1 }, xenial : { release : xenial , status : released , version : 0.17.1.1~16.04.1 }, artful : { release : artful , status : released , version : 0.17.1.1 }, bionic : { release : bionic , status : needed , version : false } } } } } Usage This is used in a few other modules (like bass in jellyfishaudits) to get information about CVEs. And is used in sherlockfish to do semi-automatic, on demand audits of information. Future This is a webscraper. If ever Ubuntu updates it's formatting we'll need to update our scraping. It's probably a good idea to instead of scraping the website go straight to the source for our future needs.","title":"Canonical CVE"},{"location":"modules/canonical_cve/#canonical-cve","text":"This is actually a simple module. Give this command a CVE number and it will query Ubuntu's CVE site to get any associated package data for the CVE in question. The following is an example with CVE-2018-8885 . ./canonical_cve.py CVE-2018-8885 | jq '.' { references : [ https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-8885 , https://launchpad.net/bugs/1753772 , https://usn.ubuntu.com/usn/usn-3607-1 ], title : CVE-2018-8885 , priority : Medium , packages : { screen-resolution-extra : { package_name : screen-resolution-extra , package_upstream_status : needs-triage , releases : { precise : { release : precise , status : dne , version : false }, trusty : { release : trusty , status : released , version : 0.17.1.1~14.04.1 }, xenial : { release : xenial , status : released , version : 0.17.1.1~16.04.1 }, artful : { release : artful , status : released , version : 0.17.1.1 }, bionic : { release : bionic , status : needed , version : false } } } } }","title":"Canonical CVE"},{"location":"modules/canonical_cve/#usage","text":"This is used in a few other modules (like bass in jellyfishaudits) to get information about CVEs. And is used in sherlockfish to do semi-automatic, on demand audits of information.","title":"Usage"},{"location":"modules/canonical_cve/#future","text":"This is a webscraper. If ever Ubuntu updates it's formatting we'll need to update our scraping. It's probably a good idea to instead of scraping the website go straight to the source for our future needs.","title":"Future"},{"location":"modules/collate/","text":"Collate Introduction We used to collate in-line in the Analyze module. However that proved to be quite costly. So we moved to a system where we analyze and then we collate our data seperately. This module does that. It figures passed, failed and exempt audits on a per-audit, per-pop and per-srvtype basis. Arch The idea here is simple. It will cycle through the various by factors (pop, srvtype audit/acoll). It counts up the results in audits_by_host for each of these factors and then update the related numbers in audits_by_factor . Future State This probably couldn't be moved to a lambda (unless the future database is quite quick) but it could be an excellent candidate for Amazon Glue as it's in effect a simplistic ETL job.","title":"Collate"},{"location":"modules/collate/#collate","text":"","title":"Collate"},{"location":"modules/collate/#introduction","text":"We used to collate in-line in the Analyze module. However that proved to be quite costly. So we moved to a system where we analyze and then we collate our data seperately. This module does that. It figures passed, failed and exempt audits on a per-audit, per-pop and per-srvtype basis.","title":"Introduction"},{"location":"modules/collate/#arch","text":"The idea here is simple. It will cycle through the various by factors (pop, srvtype audit/acoll). It counts up the results in audits_by_host for each of these factors and then update the related numbers in audits_by_factor .","title":"Arch"},{"location":"modules/collate/#future-state","text":"This probably couldn't be moved to a lambda (unless the future database is quite quick) but it could be an excellent candidate for Amazon Glue as it's in effect a simplistic ETL job.","title":"Future State"},{"location":"modules/collection_archive/","text":"Collection Archive The collections table is big and while it is (in my opinion) well indexed even well indexed tables get too big eventually. This implments archiving of older collection data. Data that has been obsolete for 7 days get's moved from the collection table to the collection_archive table. This get's ran daily as a part of the run_schedule.sh job sequence. Future It's likely that this could be made obsolete in the future with better sharding. Additionally this could probably be turned into a serverless function if ever the decision was made to migrate to the cloud.","title":"Collection Archive"},{"location":"modules/collection_archive/#collection-archive","text":"The collections table is big and while it is (in my opinion) well indexed even well indexed tables get too big eventually. This implments archiving of older collection data. Data that has been obsolete for 7 days get's moved from the collection table to the collection_archive table. This get's ran daily as a part of the run_schedule.sh job sequence.","title":"Collection Archive"},{"location":"modules/collection_archive/#future","text":"It's likely that this could be made obsolete in the future with better sharding. Additionally this could probably be turned into a serverless function if ever the decision was made to migrate to the cloud.","title":"Future"},{"location":"modules/collector/","text":"Collector By itself collector.py is actually a simple module. It simply parses some configs to tell itself what it out to do. Then runs and parses those commands on individual hosts. It's main usage is to get called by schedule2.py over and over again to do the collections. Collection Configuration Collection configuration is by default stored in svn in the collector.ini file. Here is an example of a collection : [packages] ; Remote Collection Command multi=TRUE collection=dpkg --list | grep -E '^ii' | awk '{print $2 \\011 $3}' Collections can be either single dimentional or multi-dimensional. Single dimentional lines will get stored as just one \"default\" collection subtype. Multidimentional ones will get stored with each line being treated as a subtype value combo where whitespace seperates the two items. Here are the top few results of what this command produces. Note how each package will end up being it's own subtype. : dpkg --list | grep -E '^ii' | awk '{print $2 \\011 $3}' | head accountsservice 0.6.42-0ubuntu3.1 acl 2.2.52-3build1 acpi-support 0.142 acpid 1:2.0.28-1ubuntu1 adduser 3.113+nmu3ubuntu5 adium-theme-ubuntu 0.3.4-0ubuntu4 adobe-flash-properties-gtk 1:20180313.1-0ubuntu0.17.10.1 adobe-flashplugin 1:20180313.1-0ubuntu0.17.10.1 adwaita-icon-theme 3.26.0-0ubuntu2 aisleriot 1:3.22.2-0ubuntu1 Interpolation is a thing. When the config is parsed any % character is going to be specially viewed. If you need a % character (see example below) you'll need to write a %% . You'll find it useful with stat and date commands. Here is an example of a collection that uses this: [ssh-host-key-age] ; Grabs a hash of the SSH host key multi=TRUE collection=stat -c %%n %%Y /etc/ssh/*.pub | tr '[/.]' '_' SSH Python's Paramiko module is used to log into each host and the jellyfish ssh key that get's installed in salt is generally used to log in. Currently that key is managed by hand on the jellyfish machine and it's public half is delivered via salt. Example Below is an example of a collection being ran manually and the data it produces (truncated to protect the innocent and the scrollwheels). Note that because of the uniqueness of bast servers a good chunk of collections don't work out of the box. { collection_data : { boottime : { default : Tue Mar 20 00:00:00 UTC 2018 }, cpu-info : { Architecture: : x86_64 , BogoMIPS: : 4000.00 , Byte : Order: Little Endian , CPU : MHz: 2000.000 , CPU(s): : 8 , Core(s) : per socket: 4 , Hypervisor : vendor: VMware , L1d : cache: 32K , L1i : cache: 32K , L2 : cache: 256K , L3 : cache: 15360K , Model: : 45 , NUMA : node0 CPU(s): 0-7 , On-line : CPU(s) list: 0-7 , Socket(s): : 2 , Stepping: : 7 , Thread(s) : per core: 1 , Vendor : ID: GenuineIntel , Virtualization : type: full }, host_host : { HOSTNAME : host1.host , POP : N/A , SRVTYPE : N/A , STATUS : N/A , UBERID : N/A }, pci-info : { 00:00.0 : Host bridge: Intel Corporation 440BX/ZX/DX - 82443BX/ZX/DX Host bridge (rev 01) , 00:01.0 : PCI bridge: Intel Corporation 440BX/ZX/DX - 82443BX/ZX/DX AGP bridge (rev 01) , 00:07.0 : ISA bridge: Intel Corporation 82371AB/EB/MB PIIX4 ISA (rev 08) , 00:07.1 : IDE interface: Intel Corporation 82371AB/EB/MB PIIX4 IDE (rev 01) , 00:07.3 : Bridge: Intel Corporation 82371AB/EB/MB PIIX4 ACPI (rev 08) , 00:07.7 : System peripheral: VMware Virtual Machine Communication Interface (rev 10) , 00:0f.0 : VGA compatible controller: VMware SVGA II Adapter , 00:11.0 : PCI bridge: VMware PCI bridge (rev 02) , 00:15.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 03:00.0 : Ethernet controller: VMware VMXNET3 Ethernet Controller (rev 01) }, reboot-needed : { default : NO-REBOOT-REQUIRED }, release : { default : trusty }, repositories-hash : { default : 1fe60f07c344e2a4e9459be05aed9c12+8 }, rkernel : { default : 4.4.0-103-generic } }, collection_hostname : host1.host , collection_status : SSH SUCCESS , collection_timestamp : 1522187510, connection_string : chalbersma@1.1.1.1 , ip_intel : [ { ip : 1.1.1.1 , iptype : host4 } ], ipv6_val_error : Found IPV6 Address would not Validateillegal IP address string passed to inet_pton , pop : N/A , srvtype : N/A , status : N/A , uber_id : N/A }","title":"Collector"},{"location":"modules/collector/#collector","text":"By itself collector.py is actually a simple module. It simply parses some configs to tell itself what it out to do. Then runs and parses those commands on individual hosts. It's main usage is to get called by schedule2.py over and over again to do the collections.","title":"Collector"},{"location":"modules/collector/#collection-configuration","text":"Collection configuration is by default stored in svn in the collector.ini file. Here is an example of a collection : [packages] ; Remote Collection Command multi=TRUE collection=dpkg --list | grep -E '^ii' | awk '{print $2 \\011 $3}' Collections can be either single dimentional or multi-dimensional. Single dimentional lines will get stored as just one \"default\" collection subtype. Multidimentional ones will get stored with each line being treated as a subtype value combo where whitespace seperates the two items. Here are the top few results of what this command produces. Note how each package will end up being it's own subtype. : dpkg --list | grep -E '^ii' | awk '{print $2 \\011 $3}' | head accountsservice 0.6.42-0ubuntu3.1 acl 2.2.52-3build1 acpi-support 0.142 acpid 1:2.0.28-1ubuntu1 adduser 3.113+nmu3ubuntu5 adium-theme-ubuntu 0.3.4-0ubuntu4 adobe-flash-properties-gtk 1:20180313.1-0ubuntu0.17.10.1 adobe-flashplugin 1:20180313.1-0ubuntu0.17.10.1 adwaita-icon-theme 3.26.0-0ubuntu2 aisleriot 1:3.22.2-0ubuntu1 Interpolation is a thing. When the config is parsed any % character is going to be specially viewed. If you need a % character (see example below) you'll need to write a %% . You'll find it useful with stat and date commands. Here is an example of a collection that uses this: [ssh-host-key-age] ; Grabs a hash of the SSH host key multi=TRUE collection=stat -c %%n %%Y /etc/ssh/*.pub | tr '[/.]' '_'","title":"Collection Configuration"},{"location":"modules/collector/#ssh","text":"Python's Paramiko module is used to log into each host and the jellyfish ssh key that get's installed in salt is generally used to log in. Currently that key is managed by hand on the jellyfish machine and it's public half is delivered via salt.","title":"SSH"},{"location":"modules/collector/#example","text":"Below is an example of a collection being ran manually and the data it produces (truncated to protect the innocent and the scrollwheels). Note that because of the uniqueness of bast servers a good chunk of collections don't work out of the box. { collection_data : { boottime : { default : Tue Mar 20 00:00:00 UTC 2018 }, cpu-info : { Architecture: : x86_64 , BogoMIPS: : 4000.00 , Byte : Order: Little Endian , CPU : MHz: 2000.000 , CPU(s): : 8 , Core(s) : per socket: 4 , Hypervisor : vendor: VMware , L1d : cache: 32K , L1i : cache: 32K , L2 : cache: 256K , L3 : cache: 15360K , Model: : 45 , NUMA : node0 CPU(s): 0-7 , On-line : CPU(s) list: 0-7 , Socket(s): : 2 , Stepping: : 7 , Thread(s) : per core: 1 , Vendor : ID: GenuineIntel , Virtualization : type: full }, host_host : { HOSTNAME : host1.host , POP : N/A , SRVTYPE : N/A , STATUS : N/A , UBERID : N/A }, pci-info : { 00:00.0 : Host bridge: Intel Corporation 440BX/ZX/DX - 82443BX/ZX/DX Host bridge (rev 01) , 00:01.0 : PCI bridge: Intel Corporation 440BX/ZX/DX - 82443BX/ZX/DX AGP bridge (rev 01) , 00:07.0 : ISA bridge: Intel Corporation 82371AB/EB/MB PIIX4 ISA (rev 08) , 00:07.1 : IDE interface: Intel Corporation 82371AB/EB/MB PIIX4 IDE (rev 01) , 00:07.3 : Bridge: Intel Corporation 82371AB/EB/MB PIIX4 ACPI (rev 08) , 00:07.7 : System peripheral: VMware Virtual Machine Communication Interface (rev 10) , 00:0f.0 : VGA compatible controller: VMware SVGA II Adapter , 00:11.0 : PCI bridge: VMware PCI bridge (rev 02) , 00:15.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:15.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:16.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:17.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.0 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.1 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.2 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.3 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.4 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.5 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.6 : PCI bridge: VMware PCI Express Root Port (rev 01) , 00:18.7 : PCI bridge: VMware PCI Express Root Port (rev 01) , 03:00.0 : Ethernet controller: VMware VMXNET3 Ethernet Controller (rev 01) }, reboot-needed : { default : NO-REBOOT-REQUIRED }, release : { default : trusty }, repositories-hash : { default : 1fe60f07c344e2a4e9459be05aed9c12+8 }, rkernel : { default : 4.4.0-103-generic } }, collection_hostname : host1.host , collection_status : SSH SUCCESS , collection_timestamp : 1522187510, connection_string : chalbersma@1.1.1.1 , ip_intel : [ { ip : 1.1.1.1 , iptype : host4 } ], ipv6_val_error : Found IPV6 Address would not Validateillegal IP address string passed to inet_pton , pop : N/A , srvtype : N/A , status : N/A , uber_id : N/A }","title":"Example"},{"location":"modules/scheduler/","text":"Scheduler Introduction The scheduler is a module that essentially interacts with three other modules, collector , storage and sapicheck . It ties these together and in a big ole loop enumerates the systems in Uber and attempts to profile them. Performance As of this writing, the scheduler is configured to utilize 256 threads for collecting. Because of the nature of collections (it's a lot of do wait), most of the time you spend waiting for processes to return on the target hosts. The 256 threads limitiation is likely a database limitation. Because of these limitations the scheduler is limited to run for 8 hours a day. It prioritizes systems in production and ends up behind schedule quite a bit. The orignal plan was to migrate this into EBY but before we could do that SecEng was Oathafied. So the future of the product will be decided by whomever takes it over. Input/Output The scheduler is called by cron (at the moment) and it is fenced with a lockfile in /var/run/jellyfish . The scheduler utilizes a configurable amount of threads (configured in scheduler.ini in SVN) to run a number of instances of the Collector Storage modules. It uses the server4.csv file located in the netinfo svn location to grab a list of servers it needs to check. The module will quit after a certain amount of time where it can't reach all the hosts. In this scenario it will return an item in the output json that looks like this: \"Timeout\": \"Timeout reached at 57627.98305392265 seconds with 29 items left on the queue.\", The Verbose module will output a status message to stdout for the duration of the run. Additionally a final status json will be outputted to stdout (and optionally to a seperate file) that contains the following pieces of information: global_fail_hosts - How many hosts scheduler failed to collect data from global_fail_hosts_list - A list of those hosts global_fail_prod - How many hosts scheduler failed that were listed as \"production\". global_fail_prod_list - A list of those production hosts. global_success_hosts - How many hosts scheduler successfully collected data from. global_success_hosts_list - A list of those hosts. jobtime - How long, in seconds, scheduler ran for. threads - How many threads the system used Gap Fix Gap fix is a related module to Scheduler. It's designed to find systems that are in prod in uber but don't have a recent result. It works almost the same as scheduler but has a lighter load so that it doesn't interrupt normal operations. Most importantly it doesn't trigger the lock file that keeps analysis from running. Future State Probable Ideally this would in the future be replaced with stingcell, an agent that runs on hosts and reports information back via the storage api. Scheduler would likely have a funcationality similar to gap_fix and only stand to attempt to login to systems that hadn't checked in via the api for a period of time. It would be likely that a box in EC IR500 Command and Control environments would exist with API keys that can log into the missing hosts (See Diagram). And it, like stingcell would utlize the API to make it's storage calls. While the document only shows one LC, it's likely we'd have a node in every LC that we had along with (potentially) additional nodes for Cloud Assets if that was the direction we wanted to go. In a fully realized, microservices architecture the Scheduler would be obsolete.","title":"Scheduler"},{"location":"modules/scheduler/#scheduler","text":"","title":"Scheduler"},{"location":"modules/scheduler/#introduction","text":"The scheduler is a module that essentially interacts with three other modules, collector , storage and sapicheck . It ties these together and in a big ole loop enumerates the systems in Uber and attempts to profile them.","title":"Introduction"},{"location":"modules/scheduler/#performance","text":"As of this writing, the scheduler is configured to utilize 256 threads for collecting. Because of the nature of collections (it's a lot of do wait), most of the time you spend waiting for processes to return on the target hosts. The 256 threads limitiation is likely a database limitation. Because of these limitations the scheduler is limited to run for 8 hours a day. It prioritizes systems in production and ends up behind schedule quite a bit. The orignal plan was to migrate this into EBY but before we could do that SecEng was Oathafied. So the future of the product will be decided by whomever takes it over.","title":"Performance"},{"location":"modules/scheduler/#inputoutput","text":"The scheduler is called by cron (at the moment) and it is fenced with a lockfile in /var/run/jellyfish . The scheduler utilizes a configurable amount of threads (configured in scheduler.ini in SVN) to run a number of instances of the Collector Storage modules. It uses the server4.csv file located in the netinfo svn location to grab a list of servers it needs to check. The module will quit after a certain amount of time where it can't reach all the hosts. In this scenario it will return an item in the output json that looks like this: \"Timeout\": \"Timeout reached at 57627.98305392265 seconds with 29 items left on the queue.\", The Verbose module will output a status message to stdout for the duration of the run. Additionally a final status json will be outputted to stdout (and optionally to a seperate file) that contains the following pieces of information: global_fail_hosts - How many hosts scheduler failed to collect data from global_fail_hosts_list - A list of those hosts global_fail_prod - How many hosts scheduler failed that were listed as \"production\". global_fail_prod_list - A list of those production hosts. global_success_hosts - How many hosts scheduler successfully collected data from. global_success_hosts_list - A list of those hosts. jobtime - How long, in seconds, scheduler ran for. threads - How many threads the system used","title":"Input/Output"},{"location":"modules/scheduler/#gap-fix","text":"Gap fix is a related module to Scheduler. It's designed to find systems that are in prod in uber but don't have a recent result. It works almost the same as scheduler but has a lighter load so that it doesn't interrupt normal operations. Most importantly it doesn't trigger the lock file that keeps analysis from running.","title":"Gap Fix"},{"location":"modules/scheduler/#future-state","text":"","title":"Future State"},{"location":"modules/scheduler/#probable","text":"Ideally this would in the future be replaced with stingcell, an agent that runs on hosts and reports information back via the storage api. Scheduler would likely have a funcationality similar to gap_fix and only stand to attempt to login to systems that hadn't checked in via the api for a period of time. It would be likely that a box in EC IR500 Command and Control environments would exist with API keys that can log into the missing hosts (See Diagram). And it, like stingcell would utlize the API to make it's storage calls. While the document only shows one LC, it's likely we'd have a node in every LC that we had along with (potentially) additional nodes for Cloud Assets if that was the direction we wanted to go. In a fully realized, microservices architecture the Scheduler would be obsolete.","title":"Probable"}]}
\ No newline at end of file
diff --git a/docs/search/worker.js b/docs/search/worker.js
index 5a3692c..a3ccc07 100644
--- a/docs/search/worker.js
+++ b/docs/search/worker.js
@@ -84,6 +84,7 @@ function onScriptsLoaded () {
     console.log('Lunr index built, search ready');
   }
   allowSearch = true;
+  postMessage({allowSearch: allowSearch});
 }
 
 function init () {
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index 0e39a36..a34fcf8 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -2,47 +2,47 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
      <loc>None</loc>
-     <lastmod>2018-08-09</lastmod>
+     <lastmod>2018-09-25</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2018-08-09</lastmod>
+     <lastmod>2018-09-25</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2018-08-09</lastmod>
+     <lastmod>2018-09-25</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2018-08-09</lastmod>
+     <lastmod>2018-09-25</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2018-08-09</lastmod>
+     <lastmod>2018-09-25</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2018-08-09</lastmod>
+     <lastmod>2018-09-25</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2018-08-09</lastmod>
+     <lastmod>2018-09-25</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2018-08-09</lastmod>
+     <lastmod>2018-09-25</lastmod>
      <changefreq>daily</changefreq>
     </url>
     <url>
      <loc>None</loc>
-     <lastmod>2018-08-09</lastmod>
+     <lastmod>2018-09-25</lastmod>
      <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/docs/sitemap.xml.gz b/docs/sitemap.xml.gz
index 460deff95b5d9430f7287a1241e216777fe94477..e20c7f5c0e5028bd8f20c1055f425b790f28c988 100644
GIT binary patch
literal 202
zcmV;*05$&~iwFp}Y^qxV|8r?{Wo=<_E_iKh0PU1P4uUWcMfaSd>7EuyOu&?~bLmPC
z(3A>IN{gK;dV7nY@dj>~&HVZEGjBJnc?<^Kl?Ux?jdPM=q^xwUwq1?Sr$bueh9~Ui
z4XA)(*g}o_5XL>F(=-v#K@DOA;(SLz_Hh8ZnlvL7<_U^5A6OJ(#z}s3RxvuOk0~Gw
zuH{9RmuXg|#V)Q_Z;i-a*iK!%y7E?N^JMhLSt#_2*?=2x1O6BIZt-pK7sq40tY!oN
E04kVYSpWb4

literal 200
zcmV;(05|_1iwFpUXlz>o|8r?{Wo=<_E_iKh0PU2)4uUWYhVMMZvOC-G1ZIqzM^E|y
z%b2iaY`9L*w|58{-@pq^leT~V^t+WekHMn54q&{iaX}bH+9}@}*VXuZI^-p8M8<Ey
zf{rMe9n`pwaokfnO%nkV^&m$eK6DgRp9Y|-NExZH$WV&;z|tXZy%I<7G^ca-5hcXI
zw_?YNGG{5Pv|hE<qI&5%eF^$1T50T)(;sD_&@W~KZom!rU*Nl>w;^7H6ZqI>1ONc|
Cr(Z?@

diff --git a/docs/swagger/jellyfish_swagger.yaml b/docs/swagger/jellyfish_swagger.yaml
index ce83593..c1d2426 100644
--- a/docs/swagger/jellyfish_swagger.yaml
+++ b/docs/swagger/jellyfish_swagger.yaml
@@ -288,7 +288,7 @@ paths:
         - name: audit_id
           in: path
           description: |
-            The id of the audit you wish to see. You need to either specify it in path or optionally 
+            The id of the audit you wish to see. You need to either specify it in path or optionally
             in the query string. This parameter is not technically required. There is not good way to document
             this in swagger.
           required: true
@@ -578,7 +578,7 @@ paths:
           required: false
         - name: exact
           in: query
-          description: | 
+          description: |
             Boolean use exact matches instead of regex
           schema:
             type: string
@@ -768,8 +768,8 @@ paths:
         - name: timestamp
           in: path
           description: |
-            A Unix Timestamp that you want to check for the date against. 
-            Must be a positive integer. This variable is required. If you 
+            A Unix Timestamp that you want to check for the date against.
+            Must be a positive integer. This variable is required. If you
             wish to get the "latest" you shoul utilize the /auditresults
             endpoint.
           schema:
@@ -987,7 +987,7 @@ paths:
         x-astliteraleval: true
         in: query
         description: |
-          By default this will be `4` (as in ipv4). But you can set it to 
+          By default this will be `4` (as in ipv4). But you can set it to
           `6` which will allow you to look for ipv6 ports and protocols on
           ipv6 vips.
         schema:
@@ -1424,7 +1424,7 @@ paths:
           x-astliteraleval: true
           in: query
           description: |
-            An audit ID to check against. Will filter results to just the auditID that you're interested in. For example, specifying 
+            An audit ID to check against. Will filter results to just the auditID that you're interested in. For example, specifying
             7 with a collatedType of "pop" will lead to all of the pops returning their pass/fail/exempt amounts for auditID #7.
           schema:
             type: string