[tests] Added custom main with transparent parameters for tests (#2681).

.travis.yml

@@ -71,7 +71,6 @@ matrix:
           - BUILD_TYPE=Release
           - BUILD_OPTS='-DENABLE_MONOTONIC_CLOCK=ON'
 script:
-    - TESTS_IPv6="TestMuxer.IPv4_and_IPv6:TestIPv6.v6_calls_v6*:ReuseAddr.ProtocolVersion:ReuseAddr.*6" ; # Tests to skip due to lack of IPv6 support
     - if [ "$TRAVIS_COMPILER" == "x86_64-w64-mingw32-g++" ]; then
         export CC="x86_64-w64-mingw32-gcc";
         export CXX="x86_64-w64-mingw32-g++";
@@ -95,7 +94,7 @@ script:
       fi
     - if [ "$TRAVIS_COMPILER" != "x86_64-w64-mingw32-g++" ]; then
         ulimit -c unlimited;
-        ./test-srt --gtest_filter="-$TESTS_IPv6";
+        ./test-srt -disable-ipv6;
         SUCCESS=$?;
         if [ -f core ]; then gdb -batch ./test-srt -c core -ex bt -ex "info thread" -ex quit; else echo "NO CORE - NO CRY!"; fi;
         test $SUCCESS == 0;

test/TESTS_HOWTO.md

@@ -0,0 +1,107 @@
+# Rules for writing tests for SRT
+
+## 1. Use automatic startup/cleanup management
+
+Note that most of the test require SRT library to be initialized for the
+time of running the test. There are two methods how you can do it:
+
+* In a free test (`TEST` macro), declare this in the beginning:
+`srt::TestInit srtinit;`
+
+* In a fixture (`TEST_F` macro), draw your class off `srt::Test`
+(instead of `testing::Test`)
+
+In the fixture case you should also use names `setup/teardown` instead of
+`SetUp/TearDown`. Both these things will properly initialize and destroy the
+library resources.
+
+## 2. Do not misuse ASSERT macros
+
+**Be careful** where you are using `ASSERT_*` macros. In distinction to
+`EXPECT_*` macros, they interrupt the testing procedure by throwing an exception.
+This means that if this fires, nothing will be executed up to the end of the
+current testing procedure, unless it's a destructor of some object constructed
+inside the procedure.
+
+This means, however, that if you have any resource deallocation procedures, which
+must be placed there for completion regardless of the test result, the call to
+`ASSERT_*` macro will skip them, which may often lead to misexecution of the
+remaining tests and have them falsely failed. If this interruption is necessary,
+there are the following methods you can use to prevent skipping resource cleanup:
+
+* Do not cleanup anything in the testing procedure. Use the fixture's teardown
+method for any cleaning. Remember also that it is not allowed to use `ASSERT_*`
+macros in the teardown procedure, should you need to test additionally to the
+cleanup.
+
+* You can also use a local class with a destructor so that cleanups will execute
+no matter what happened inside the procedure
+
+* Last resort, keep the code that might use `ASSERT_*` macro in the try-catch
+block and free the resources in the `catch` clause, then rethrow the exception.
+A disadvantage of this solution is that you'll have to repeat the cleanup
+procedure outside the try-catch block.
+
+* Use `EXPECT_` macros, but still check the condition again and skip required
+parts of the test that could not be done without this resource.
+
+# Useful SRT test features
+
+## Test command line parameters
+
+The SRT tests support command-line parameters. They are available in test
+procedures, startups, and through this you can control some execution
+aspects. The gtest-specific options are being removed from the command
+line by the gtest library itself; all other parameters are available for
+the user. The main API access function for this is `srt::TestEnv`. This
+is a fixed singleton object accessed through `srt::TestEnv::me` pointer.
+These arguments are accessible through two fields:
+
+* `TestEnv::args`: a plain vector with all the command line arguments
+* `TestEnv::argmap`: a map of arguments parsed according to the option syntax
+
+The option syntax is the following:
+
+* `-option` : single option without argument; can be tested for presence
+* `-option param1 param2 param3` : multiple parameters assigned to an option
+
+Special markers:
+
+* `--`: end of options
+* `-/`: end of parameters for the current option
+
+To specify free parameters after an option (and possibly its own parameters),
+end the parameter list with the `-/` phrase. The `--` phrase means that the
+rest of command line parameters are arguments for the last specified option,
+even if they start with a dash. Note that a single dash has no special meaning.
+
+The `TestEnv::argmap` is using option names (except the initial dash) as keys
+and the value is a vector of the parameters specified after the option. Free
+parameters are collected under an empty string key. For convenience you can
+also use two `TestEnv` helper methods:
+
+* `OptionPresent(name)`: returns true if the option of `name` is present in the
+map (note that options without parameters have simply an empty vector assigned)
+
+* `OptionValue(name)`: returns a string that contains all parameters for that
+option separated by a space (note that the value type in the map is a vector
+of strings)
+
+## Test environment feature checks
+
+The macro `SRTST_REQUIRE` can be used to check if particular feature of the
+test environment is available. This binds to the `TestEnv::Available_FEATURE`
+option if used as `SRTST_REQUIRE(FEATURE)`. This macro makes the test function
+exit immediately with success. The checking function should take care of
+printing appropriate information about that the test was forcefully passed.
+
+To add more environment availability features, add more `TestEnv::Available_*`
+methods. Methods must return `bool`, but may have parameters, which are passed
+next to the first argument in the macro transparently. Availability can be
+tested internally, or taken as a good deal basing on options, as it is
+currently done with the IPv6 feature - it is declared as not available when the
+test application gets the `-disable-ipv6` option.
+
+It is unknown what future tests could require particular system features,
+so this solution is open for further extensions.
+

test/filelist.maf

@@ -1,7 +1,9 @@
 HEADERS
 any.hpp
+test_env.h
 
 SOURCES
+test_main.cpp
 test_buffer_rcv.cpp
 test_common.cpp
 test_connection_timeout.cpp

test/test_bonding.cpp

@@ -5,16 +5,16 @@
 #include <functional>
 
 #include "gtest/gtest.h"
+#include "test_env.h"
 
 #include "srt.h"
 #include "netinet_any.h"
 
 TEST(Bonding, SRTConnectGroup)
 {
+    srt::TestInit srtinit;
     struct sockaddr_in sa;
 
-    srt_startup();
-
     const int ss = srt_create_group(SRT_GTYPE_BROADCAST);
     ASSERT_NE(ss, SRT_ERROR);
 
@@ -54,8 +54,6 @@ TEST(Bonding, SRTConnectGroup)
     {
         std::cerr << "srt_close: " << srt_getlasterror_str() << std::endl;
     }
-
-    srt_cleanup();
 }
 
 #define ASSERT_SRT_SUCCESS(callform) ASSERT_NE(callform, -1) << "SRT ERROR: " << srt_getlasterror_str()
@@ -133,8 +131,8 @@ void ConnectCallback(void* /*opaq*/, SRTSOCKET sock, int error, const sockaddr*
 
 TEST(Bonding, NonBlockingGroupConnect)
 {
-    srt_startup();
-    
+    srt::TestInit srtinit;
+
     const int ss = srt_create_group(SRT_GTYPE_BROADCAST);
     ASSERT_NE(ss, SRT_ERROR);
     std::cout << "Created group socket: " << ss << '\n';
@@ -207,8 +205,6 @@ TEST(Bonding, NonBlockingGroupConnect)
     listen_promise.wait();
 
     EXPECT_EQ(srt_close(ss), 0) << "srt_close: %s\n" << srt_getlasterror_str();
-
-    srt_cleanup();
 }
 
 void ConnectCallback_Close(void* /*opaq*/, SRTSOCKET sock, int error, const sockaddr* /*peer*/, int token)
@@ -226,8 +222,8 @@ void ConnectCallback_Close(void* /*opaq*/, SRTSOCKET sock, int error, const sock
 
 TEST(Bonding, CloseGroupAndSocket)
 {
-    srt_startup();
-    
+    srt::TestInit srtinit;
+
     const int ss = srt_create_group(SRT_GTYPE_BROADCAST);
     ASSERT_NE(ss, SRT_ERROR);
     std::cout << "Created group socket: " << ss << '\n';
@@ -332,7 +328,5 @@ TEST(Bonding, CloseGroupAndSocket)
     std::cout << "CLOSED GROUP. Now waiting for sender to exit...\n";
     sender.join();
     listen_promise.wait();
-
-    srt_cleanup();
 }
 

test/test_common.cpp

@@ -2,6 +2,7 @@
 #include <stdlib.h>
 
 #include "gtest/gtest.h"
+#include "test_env.h"
 #include "utilities.h"
 #include "common.h"
 
@@ -43,6 +44,7 @@ void test_cipaddress_pton(const char* peer_ip, int family, const uint32_t (&ip)[
 // Example IPv4 address: 192.168.0.1
 TEST(CIPAddress, IPv4_pton)
 {
+    srt::TestInit srtinit;
     const char*    peer_ip = "192.168.0.1";
     const uint32_t ip[4]   = {htobe32(0xC0A80001), 0, 0, 0};
     test_cipaddress_pton(peer_ip, AF_INET, ip);
@@ -51,6 +53,7 @@ TEST(CIPAddress, IPv4_pton)
 // Example IPv6 address: 2001:db8:85a3:8d3:1319:8a2e:370:7348
 TEST(CIPAddress, IPv6_pton)
 {
+    srt::TestInit srtinit;
     const char*    peer_ip = "2001:db8:85a3:8d3:1319:8a2e:370:7348";
     const uint32_t ip[4]   = {htobe32(0x20010db8), htobe32(0x85a308d3), htobe32(0x13198a2e), htobe32(0x03707348)};
 
@@ -59,9 +62,10 @@ TEST(CIPAddress, IPv6_pton)
 
 // Example IPv4 address: 192.168.0.1
 // Maps to IPv6 address: 0:0:0:0:0:FFFF:192.168.0.1
-// Simplified: 	                 ::FFFF:192.168.0.1
+// Simplified:                   ::FFFF:192.168.0.1
 TEST(CIPAddress, IPv4_in_IPv6_pton)
 {
+    srt::TestInit srtinit;
     const char*    peer_ip = "::ffff:192.168.0.1";
     const uint32_t ip[4]   = {0, 0, htobe32(0x0000FFFF), htobe32(0xC0A80001)};
 

test/test_connection_timeout.cpp

@@ -1,5 +1,6 @@
-#include <gtest/gtest.h>
 #include <chrono>
+#include <gtest/gtest.h>
+#include "test_env.h"
 
 #ifdef _WIN32
 #define INC_SRT_WIN_WINTIME // exclude gettimeofday from srt headers
@@ -16,7 +17,7 @@ using namespace std;
 
 
 class TestConnectionTimeout
-    : public ::testing::Test
+    : public ::srt::Test
 {
 protected:
     TestConnectionTimeout()
@@ -32,10 +33,8 @@ class TestConnectionTimeout
 protected:
 
     // SetUp() is run immediately before a test starts.
-    void SetUp() override
+    void setup() override
     {
-        ASSERT_EQ(srt_startup(), 0);
-
         m_sa.sin_family = AF_INET;
         m_sa.sin_addr.s_addr = INADDR_ANY;
         m_udp_sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
@@ -60,12 +59,11 @@ class TestConnectionTimeout
         ASSERT_EQ(inet_pton(AF_INET, "127.0.0.1", &m_sa.sin_addr), 1);
     }
 
-    void TearDown() override
+    void teardown() override
     {
         // Code here will be called just after the test completes.
         // OK to throw exceptions from here if needed.
-        ASSERT_NE(closesocket(m_udp_sock), -1);
-        srt_cleanup();
+        EXPECT_NE(closesocket(m_udp_sock), -1);
     }
 
 protected:

test/test_enforced_encryption.cpp

@@ -10,10 +10,11 @@
  *             Haivision Systems Inc.
  */
 
-#include <gtest/gtest.h>
 #include <thread>
 #include <condition_variable> 
 #include <mutex>
+#include <gtest/gtest.h>
+#include "test_env.h"
 
 #include "srt.h"
 #include "sync.h"
@@ -214,7 +215,7 @@ const TestCaseBlocking g_test_matrix_blocking[] =
 
 
 class TestEnforcedEncryption
-    : public ::testing::Test
+    : public srt::Test
 {
 protected:
     TestEnforcedEncryption()
@@ -230,10 +231,8 @@ class TestEnforcedEncryption
 protected:
 
     // SetUp() is run immediately before a test starts.
-    void SetUp()
+    void setup() override
     {
-        ASSERT_EQ(srt_startup(), 0);
-
         m_pollid = srt_epoll_create();
         ASSERT_GE(m_pollid, 0);
 
@@ -254,13 +253,12 @@ class TestEnforcedEncryption
         ASSERT_NE(srt_epoll_add_usock(m_pollid, m_listener_socket, &epoll_out), SRT_ERROR);
     }
 
-    void TearDown()
+    void teardown() override
     {
         // Code here will be called just after the test completes.
         // OK to throw exceptions from here if needed.
-        ASSERT_NE(srt_close(m_caller_socket),   SRT_ERROR);
-        ASSERT_NE(srt_close(m_listener_socket), SRT_ERROR);
-        srt_cleanup();
+        EXPECT_NE(srt_close(m_caller_socket),   SRT_ERROR) << srt_getlasterror_str();
+        EXPECT_NE(srt_close(m_listener_socket), SRT_ERROR) << srt_getlasterror_str();
     }