Merge pull request #57 from dmuhamedagic/vagrant_update

dev: systemtests: make setting images right (-N and -C)

cma/systemtests/README.md

@@ -27,7 +27,7 @@ to the log file. Or just do this:
 The script to start the tests is runtests.sh. Example use for
 vagrant:
 
-	$ sh runtests.sh -l /var/log/assim.log -m vagrant -C cma -N drone 20
+	$ sh runtests.sh -l /var/log/assim.log -m vagrant -C generic/ubuntu1604 -N debian/stretch64 20
 
 Example for docker:
 
@@ -45,36 +45,40 @@ Vagrant
 -------
 
 The vagrant's Vagrantfile as well as provisioning shell scripts
-are in the vagrant directory. The base box is Debian Stretch. We
-used this one:
+are in the vagrant directory. The scripts were developed on a
+Debian Stretch release. They work for most of Ubuntu releases
+such as xenial or artful. For other Debian/Ubuntu releases minor
+modifications may be necessary.
 
-https://app.vagrantup.com/debian/boxes/stretch64
+The provider is libvirt, so install the vagrant-libvirt plugin.
 
-The provider to use is libvirt, so install the vagrant-libvirt
-plugin.
+The timezone is set with the vagrant-timezone plugin.
 
-To run the tests as a regular user, add yourself to the following
+The Vagrantfile features also the configuration for
+apt-cacher-ng. It is not strictly required, but reduces runtime
+considerably.
+
+To run the tests as a regular user, add the user to the following
 groups: kvm libvirt libvirt-qemu.
 
-Before running the tests, copy the cma, nanoprobe, and libsodium
-debian packages to the vagrant directory. They are going to be
-installed in VMs from that directory.
+Here are the commands you may have to run to setup the
+environment:
 
-The VMs are going to be "cma" (for the cma) and "drone%n"
-(for drones) where "%n" stands for a drone number (a
-small integer). The drone VMs count starts at 1.
+$ sudo usermod -a -G kvm,libvirt,libvirt-qemu <user>
+$ sudo apt-get install vagrant-libvirt apt-cacher-ng
+$ vagrant plugin install vagrant-timezone
 
-The Vagrantfile contains also configuration for the
-apt-cacher-ng. It is not strictly required, but reduces runtime
-considerably.
+Before running the tests, copy the cma, nanoprobe, and libsodium
+debian packages to a subdirectories named after test boxes
+provided with "-C" and "-N" options. For instance, for
+debian/stretch64:
 
-To use other boxes/distributions please modify the Vagrantfile
-and the provisioning scripts accordingly. For other Debian based
-distributions such as Ubuntu, it _should_ be enough just to
-modify the base box name. Best to make copies in another
-directory and then use the "-D" option. It is out of scope for
-this document to describe vagrant, but it should not be very
-difficult.
+$ cd vagrant
+$ mkdir -p debian/stretch64
+
+The VMs are going to be named "cma" (for the cma) and "drone%n"
+(for drones/nanoprobes) where "%n" stands for a drone number (a
+small integer). The drone VMs count starts at 1.
 
 Vagrant ssh command is quite slow. To access VMs with ssh or pdsh
 do the following:
@@ -83,7 +87,7 @@ $ cd vagrant
 $ . mksshconf.sh
 $ ssh -F ssh_config <hostname>
 
-To run commands on all VMs, install pdsh:
+To run commands on all VMs with pdsh:
 
 $ pdsh date
 drone1: Wed Mar 28 15:30:30 UTC 2018

cma/systemtests/assimtest.py

@@ -219,6 +219,8 @@ def testmain(logname):
         testset = [test for test in AssimSysTest.testset]
 
     # Use cmaimage for nanoimages as default
+    # (this doesn't work because nanoimages is always set to the
+    # default)
     if len(opts.nanoimages) > 0:
         nanoimages=opts.nanoimages.split()
     else:

cma/systemtests/sysmgmt.py

@@ -50,7 +50,7 @@ class TestSystem(object):
 
     def __init__(self, imagename, imagecount=2, cmdargs=None):
         'Constructor for Abstract class TestSystem'
-        self.mkname(imagename)
+        self.mkname()
         TestSystem.nameindex += 1
         if TestSystem.tmpdir is None:
             TestSystem.tmpdir = tempfile.mkdtemp(TestSystem.tmpsuffix
@@ -157,7 +157,7 @@ def __del__(self):
         "Invoke our destroy operation when we're deleted"
         #self.destroy()
 
-    def mkname(self, imagename):
+    def mkname(self):
         self.name = TestSystem.nameformat % (self.__class__.__name__, os.getpid()
         ,   TestSystem.nameindex)
 
@@ -283,6 +283,7 @@ def __init__(self, imagename, imagecount=2, cmdargs=None, cleanupwhendone=False)
         # need to pass the number of nanoprobe VMs to Vagrant
         os_env = os.environ.copy()
         os_env['NUM_DRONES'] = str(imagecount)
+        os_env['ASSIM_TEST_BOX'] = imagename
         self.v = vagrant.Vagrant(env=os_env, quiet_stdout=False, quiet_stderr=False)
         self.hostname = 'unknown'
         self.ipaddr = 'unknown'
@@ -294,11 +295,11 @@ def __del__(self):
         "Invoke our destroy operation when we're deleted"
         #self.destroy()
 
-    def mkname(self, imagename):
-        if imagename == "drone":
+    def mkname(self):
+        if TestSystem.nameindex > 0:
             self.name = "drone%d" % TestSystem.nameindex
         else:
-            self.name = imagename
+            self.name = "cma"
 
     def start(self):
         'Start a vagrant instance'

cma/systemtests/vagrant/Vagrantfile

@@ -19,16 +19,21 @@ else
 end
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "debian/stretch64"
+  config.vm.box = test_box
   config.vm.box_check_update = false
+  if Vagrant.has_plugin?("vagrant-timezone")
+	  config.timezone.value = :host
+  end
   config.apt_proxy.http = "http://192.168.121.1:3142"
   #config.apt_proxy.http = "http://10.0.2.2:3142"
   config.apt_proxy.https = "DIRECT"
   config.vm.provider "libvirt"
+  config.vm.synced_folder test_box, '/vagrant', type: '9p', readonly: true
   config.vm.define "cma" do |cma|
 	  cma.vm.hostname = "cma"
 	  cma.vm.provision :shell, path: "install_nanoprobe"
 	  cma.vm.provision :shell, path: "install_cma"
+	  cma.vm.provision :shell, path: "configure_cma"
 	  cma.vm.provision :shell, path: "common"
 	  config.vm.network :forwarded_port, guest: 7474, host: 17474
   end

cma/systemtests/vagrant/configure_cma

@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+
+echo "dbms.connector.http.address=0.0.0.0:7474" >> /etc/neo4j/neo4j.conf
+echo "cypher.forbid_shortestpath_common_nodes=false" >> /etc/neo4j/neo4j.conf
+# py2neo wants the config elsewhere
+mkdir -p /usr/share/neo4j/conf
+ln -s /etc/neo4j/neo4j.conf /usr/share/neo4j/conf
+chown -R neo4j /usr/share/neo4j/conf
+rm -fr /var/lib/neo4j/data/databases/graph.db /var/lib/neo4j/data/dbms/auth
+service neo4j start; sleep 10
+assimcli genkeys
+rm -f "/usr/share/assimilation/crypto.d/#CMA#00001.secret"
+assimcli neo4jpass neo4j2
+service neo4j stop

cma/systemtests/vagrant/install_cma

@@ -14,19 +14,3 @@ echo "deb http://debian.neo4j.org/repo ${neoversion}/" > /etc/apt/sources.list.d
 apt-get -y update
 apt-get -y install --no-install-recommends neo4j=$NEOREL
 apt-get -y install --no-install-recommends /vagrant/assimilation-cma_*.deb
-
-echo "dbms.connector.http.address=0.0.0.0:7474" >> /etc/neo4j/neo4j.conf
-echo "cypher.forbid_shortestpath_common_nodes=false" >> /etc/neo4j/neo4j.conf
-# py2neo wants the config elsewhere
-mkdir -p /usr/share/neo4j/conf
-ln -s /etc/neo4j/neo4j.conf /usr/share/neo4j/conf
-chown -R neo4j /usr/share/neo4j/conf
-rm -fr /var/lib/neo4j/data/databases/graph.db /var/lib/neo4j/data/dbms/auth
-service neo4j start; sleep 10
-assimcli genkeys
-rm -f "/usr/share/assimilation/crypto.d/#CMA#00001.secret"
-assimcli neo4jpass neo4j2
-service neo4j stop
-
-# create keys tarball for drones
-tar -cf cma_pubkeys.tar /usr/share/assimilation/crypto.d/*CMA*.pub