Plugin-sandbox-based OUTCONTAINERSAFE Mode

.coveragerc

@@ -10,3 +10,4 @@ exclude_lines =
 ignore_errors = True
 omit =
     tests/*
+    crawler/utils/plugincont/*

.travis.yml

@@ -2,7 +2,6 @@ sudo: required
 dist: trusty
 group: deprecated-2017Q4
 language: python
-
 services:
   - docker
 
@@ -35,6 +34,8 @@ before_install:
     - cp -r psvmi/maps maps
     - cp -r psvmi/offsets offsets
     - cp psvmi/header.h .
+    # for safe plugin mode
+    - sudo apt-get install libcap-dev iptables iptables-dev
 
 # command to install dependencies
 # XXX: Now mock complains if we don't `sudo pip install`.

Dockerfile

@@ -14,6 +14,9 @@ COPY \
 RUN dpkg -i /tmp/python-socket-datacollector_*_all.deb && \
     apt-get -y update && \
     apt-get -y install libpcap0.8 && \
+    apt-get -y install libcap-dev && \
+    apt-get -y install iptables && \
+    apt-get -y install iptables-dev && \
     dpkg -i /tmp/softflowd_0.9.*_amd64.deb && \
     pip install pyroute2 py-radix requests-unixsocket json-rpc && \
     dpkg -i /tmp/python-conntrackprobe_*_all.deb && \

crawler/.gitignore

@@ -4,5 +4,4 @@ binaries/
 kafka-producer.py
 timeout.py
 alchemy.py
-*.json
 *.sh

crawler/containers.py

@@ -10,7 +10,8 @@
 
 
 def list_all_containers(user_list='ALL', host_namespace='',
-                        ignore_raw_containers=True):
+                        ignore_raw_containers=True,
+                        group_by_pid_namespace=True):
     """
     Returns a list of all running containers in the host.
 
@@ -24,10 +25,13 @@ def list_all_containers(user_list='ALL', host_namespace='',
 
     for _container in get_docker_containers(host_namespace=host_namespace,
                                             user_list=user_list):
-        curr_ns = _container.process_namespace
-        if curr_ns not in visited_ns:
-            visited_ns.add(curr_ns)
+        if group_by_pid_namespace is False:
             yield _container
+        else:
+            curr_ns = _container.process_namespace
+            if curr_ns not in visited_ns:
+                visited_ns.add(curr_ns)
+                yield _container
 
     # XXX get list of rkt containers
 
@@ -62,7 +66,8 @@ def get_containers(
     environment='cloudsight',
     host_namespace=misc.get_host_ipaddr(),
     user_list='ALL',
-    ignore_raw_containers=True
+    ignore_raw_containers=True,
+    group_by_pid_namespace=True
 ):
     """
     Returns a list of all containers running in the host.
@@ -79,7 +84,8 @@ def get_containers(
     """
     filtered_list = []
     containers_list = list_all_containers(user_list, host_namespace,
-                                          ignore_raw_containers)
+                                          ignore_raw_containers,
+                                          group_by_pid_namespace)
     for _container in containers_list:
         default_environment = 'cloudsight'
         if (environment != default_environment and

crawler/crawler.conf

@@ -17,9 +17,9 @@
 
     [[ process_host ]]		
 
-    [[ ruby_pkg ]]		
+    [[ rubypackage ]]		
 
-    [[ python_pkg ]]		
+    [[ pythonpackage ]]		
     avoid_setns = False
 
     [[ fprobe_container ]]

crawler/crawler.py

@@ -7,6 +7,7 @@
 
 from worker import Worker
 from containers_crawler import ContainersCrawler
+from safe_containers_crawler import SafeContainersCrawler
 from utils import misc
 from crawlmodes import Modes
 from emitters_manager import EmittersManager
@@ -93,11 +94,12 @@ def main():
             Modes.OUTVM,
             Modes.MOUNTPOINT,
             Modes.OUTCONTAINER,
+            Modes.OUTCONTAINERSAFE,
             Modes.MESOS,
         ],
         default=Modes.INVM,
         help='The crawler mode: '
-             '{INVM,OUTVM,MOUNTPOINT,OUTCONTAINER}. '
+             '{INVM,OUTVM,MOUNTPOINT,OUTCONTAINER,OUTCONTAINERSAFE}. '
              'Defaults to INVM',
     )
     parser.add_argument(
@@ -222,6 +224,15 @@ def main():
             host_namespace=args.namespace,
             plugin_places=args.plugin_places,
             options=options)
+    elif args.crawlmode == 'OUTCONTAINERSAFE':
+        crawler = SafeContainersCrawler(
+            features=args.features,
+            environment=args.environment,
+            user_list=args.crawlContainers,
+            host_namespace=args.namespace,
+            plugin_places=args.plugin_places,
+            frequency=args.frequency,
+            options=options)
     else:
         raise NotImplementedError('Invalid crawlmode')
 

crawler/crawlmodes.py

@@ -4,4 +4,5 @@
              OUTVM='OUTVM',
              MOUNTPOINT='MOUNTPOINT',
              OUTCONTAINER='OUTCONTAINER',
+             OUTCONTAINERSAFE='OUTCONTAINERSAFE',
              MESOS='MESOS')

crawler/dockercontainer.py

@@ -162,7 +162,7 @@ def __init__(
         self.volumes = inspect.get('Volumes')
         self.image_name = inspect['Config']['Image']
         self.inspect = inspect
-
+        self.plugincont = None
         self.process_namespace = (process_namespace or
                                   namespace.get_pid_namespace(self.pid))