Merge pull request #651 from OpenGeoscience/webgl-testing

Use firefox and xvfb to test webgl and examples.

.travis.yml

@@ -1,8 +1,30 @@
 language: node_js
-sudo: false
+# At this time, the travis trusty sudo environment works, but the sudo: false
+# environment doesn't (it might with a bunch of apt packages).
+sudo: required
+dist: trusty
+
 node_js:
   - '0.12'
 
+addons:
+  firefox: latest
+  apt:
+    packages:
+      # I suspect that not all of these are necessary
+      - mesa-utils
+      - xvfb
+      - libosmesa6
+
+      - libgif-dev
+      - libpng-dev
+
+      - freeglut3-dev
+      - libxmu-dev
+      - libxi-dev
+      - libxxf86vm-dev
+      - libxrandr-dev
+
 cache:
   directories:
     - node_modules
@@ -12,6 +34,10 @@ before_install:
   - CACHE="${HOME}/cache" CMAKE_VERSION=3.5.0 CMAKE_SHORT_VERSION=3.5 source ./scripts/install_cmake.sh
   - npm prune
 
+before_script:
+  - export DISPLAY=:99.0
+  - sh -e /etc/init.d/xvfb start
+
 script:
   - npm run build
   - npm run docs

CMakeLists.txt

@@ -7,6 +7,8 @@ enable_testing()
 
 set(BUILD_TESTING ON CACHE BOOL "Enable geojs testing")
 set(PHANTOMJS_TESTS ON CACHE BOOL "Generate phantomjs unit tests.")
+set(FFHEADLESS_TESTS ON CACHE BOOL "Generate headless Firefox unit tests (requires xvfb to be running).")
+set(TEST_SAVE_IMAGE "none" CACHE STRING "Save headless test images even if there aren't errors.  Valid options are none, all, or the a comma-separated list of test names.")
 set(ESLINT_TESTS ON CACHE BOOL "Generate eslint style tests for JS source files.")
 set(SELENIUM_TESTS ON CACHE BOOL "Generate selenium unit tests.")
 
@@ -70,6 +72,13 @@ add_test(
 )
 set_property(TEST "notes-report" APPEND PROPERTY DEPENDS "notes-reset")
 
+add_test(
+  NAME "total-coverage"
+  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+  COMMAND npm run combine-coverage
+)
+set_property(TEST "notes-report" APPEND PROPERTY DEPENDS "notes-reset")
+
 if(PHANTOMJS_TESTS)
   find_program(NPM_EXECUTABLE npm)
   add_test(
@@ -80,8 +89,52 @@ if(PHANTOMJS_TESTS)
   set_property(TEST "phantomjs" APPEND PROPERTY ENVIRONMENT "CTEST_NOTES_PATH=${CMAKE_CURRENT_BINARY_DIR}/notes")
   set_property(TEST "phantomjs" APPEND PROPERTY DEPENDS "notes-reset")
   set_property(TEST "notes-report" APPEND PROPERTY DEPENDS "phantomjs")
+  set_property(TEST "total-coverage" APPEND PROPERTY DEPENDS "phantomjs")
+endif()
+
+add_custom_target(
+  data_files
+  ALL
+  DEPENDS ${Girder_DOWNLOAD_FILES}
+)
+add_test(NAME get_data_files COMMAND "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target data_files)
+
+if(FFHEADLESS_TESTS)
+  find_program(NPM_EXECUTABLE npm)
+  add_test(
+    NAME "ffheadless"
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    COMMAND npm run ffci
+  )
+  set_property(TEST "ffheadless" APPEND PROPERTY ENVIRONMENT "CTEST_IMAGE_PATH=${CMAKE_CURRENT_BINARY_DIR}/images")
+  set_property(TEST "ffheadless" APPEND PROPERTY ENVIRONMENT "TEST_SAVE_IMAGE=${TEST_SAVE_IMAGE}")
+  set_property(TEST "total-coverage" APPEND PROPERTY DEPENDS "ffheadless")
+  set_property(TEST "ffheadless" APPEND PROPERTY DEPENDS "get_data_files")
+
+  configure_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/testing/test-runners/baseline_images.py"
+    "${CMAKE_CURRENT_BINARY_DIR}/test/baseline_images.py"
+    COPYONLY
+  )
 endif()
 
+add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/base-images.tgz"
+  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+  # Make sure we have the data files.
+  COMMAND "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target data_files
+  # Run the ffheadless test, asking to save all images
+  COMMAND TEST_SAVE_IMAGE=all npm run ffci
+  # Make a tarball of all of the images
+  COMMAND tar -zcvf "${CMAKE_CURRENT_BINARY_DIR}/base-images.tgz" --exclude=*-test.png --exclude=*-diff.png --exclude=*-base.png -C "${CMAKE_CURRENT_BINARY_DIR}/images" .
+  COMMENT "Create baseline images, then tar them into a single file"
+  VERBATIM
+)
+
+add_custom_target(baseline_images DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/base-images.tgz")
+
+add_test(NAME baseline_images CONFIGURATIONS "baseline_images" COMMAND "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target baseline_images)
+set_property(TEST "baseline_images" APPEND PROPERTY ENVIRONMENT "CTEST_IMAGE_PATH=${CMAKE_CURRENT_BINARY_DIR}/images")
+
 if(SELENIUM_TESTS)
 
   find_package(PythonInterp REQUIRED)

cmake/travis_build.cmake

@@ -5,7 +5,7 @@ include(${CTEST_SOURCE_DIRECTORY}/CTestConfig.cmake)
 set(CTEST_SITE "Travis")
 set(CTEST_BUILD_NAME "Linux-$ENV{TRAVIS_BRANCH}")
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
-set(coverage_file "${CTEST_SOURCE_DIRECTORY}/dist/cobertura/phantomjs/coverage.xml")
+set(coverage_file "${CTEST_SOURCE_DIRECTORY}/dist/cobertura/cobertura-coverage.xml")
 
 ctest_start("Continuous")
 ctest_configure(
@@ -15,6 +15,7 @@ ctest_build()
 ctest_test(PARALLEL_LEVEL 1 RETURN_VALUE res)
 if(EXISTS "${coverage_file}")
   file(COPY "${coverage_file}" DESTINATION "${CTEST_BINARY_DIRECTORY}")
+  file(RENAME ${CTEST_BINARY_DIRECTORY}/cobertura-coverage.xml ${CTEST_BINARY_DIRECTORY}/coverage.xml)
   ctest_coverage()
   file(REMOVE ${CTEST_BINARY_DIRECTORY}/coverage.xml)
 endif()

docs/baseline_images.rst

@@ -0,0 +1,5 @@
+baseline_images
+===============
+
+.. automodule:: baseline_images
+   :members:

docs/developers.rst

@@ -7,32 +7,8 @@ Developer's guide
     This guide assumes you have cloned and built the geojs repository
     according to the :ref:`Quick start guide <project-setup-guide>`.
 
-The selenium testing infrastructure of Geojs is run via CTest, it assumes
-that the testing "server" is started prior to execution.  To start the
-server, just run ::
-
-    npm run start-test
-
-This will start a server on the default port of ``30100``.  The port
-and selenium host names are configurable with cmake.  For example inside
-the Kitware firewall, you can run the following to test on the selenium
-node on ``garant`` ::
-
-    cmake -DSELENIUM_TESTS=ON -DSELENIUM_HOST=garant /path/to/geojs
-    make
-    ctest -VV
-
-You may need to also set the variable ``TESTING_HOST`` to your computer's
-IP address reachable by the selenium node.
-
-.. note::
-
-    Typically, CMake is used to build outside of the source tree.  This
-    means you would create a new directory somewhare and point cmake
-    to the geojs source directory.  You may need to rerun ``cmake`` and
-    ``make`` after making changes to your code for everything to
-    build correctly.  Try running ``ccmake /path/to/geojs`` for a full
-    list of configuration options.
+    To run all of the tests, you will need the optional packages and python
+    modules listed there.
 
 Geojs employs several different frameworks for unit testing.  These
 frameworks have been designed to make it easy for developers to
@@ -49,6 +25,15 @@ tests are preformed automatically for every file added to the build; no
 additional configuration is required.  You can run a quick check of the
 code style outside of CMake by running ``npm run lint``.
 
+Code coverage
+-------------
+
+Code coverage information is generated automatically for all headless unit tests
+by Karma's test runner when running ``npm run test``.  The coverage information is
+submitted to `codecov <https://codecov.io/github/OpenGeoscience/geojs>`_ and
+`cdash <http://my.cdash.org/index.php?project=geojs>`_ after every
+successful Travis run.
+
 Headless browser testing
 ------------------------
 
@@ -78,9 +63,73 @@ instrumentation after running.  Ideally, each test should be runnable
 independently and use jasmines ``beforeEach`` and ``afterEach`` methods
 for setup and tear down.
 
+Headless WebGL testing
+----------------------
+
+To fully test code that uses WebGL, a browser with WebGL is required.
+If xvfb, osmesa, and Firefox are installed, some tests can be run in a virtual
+frame buffer that doesn't require a display.  May of these tests depend on
+additional data which can be downloaded by using CMake and running ctest.
+
+For example, running ::
+
+    cmake /path/to/geojs
+    make
+    xvfb-run -s '-ac -screen 0 1280x1024x24' ctest -VV -R ffheadless
+
+will run the headless WebGL tests.  After the data for tests is downloaded,
+the tests can also be run via ``npm run test-webgl``, which assumes that
+``xvfb-run`` is available.
+
+The headless unit tests that require WebGL should be placed in the
+``tests/gl-cases/`` directory.  When tests are run in a normal browser via
+``npm run start``, the webgl tests are included.
+
+Many of these tests compare against a baseline image.  If a test is changed or
+added, new baselines can be generated and optionally uploaded via the script
+built into ``test/baseline_images.py``.
+
+If a test fails, the specific test will be reported by the test runner, and the
+base and test images are saved in the ``images`` subdirectory of the build
+directory.  The images have the base name of the test and end in ``-base.png``
+for the reference image, ``-test.png`` for the current test, and ``-diff.png``
+for a difference image where areas that are different are highlight (using
+resemblejs, the default highlight color is pink).
+
+Unless an image comparison test fails, images are not automatically saved.  To
+save all images, add the environment variable ``TEST_SAVE_IMAGE=all`` to the
+test command or set this parameter in CMake.
+
+.. note::
+
+    Typically, CMake is used to build outside of the source tree.  This
+    means you would create a new directory somewhere and point cmake
+    to the geojs source directory.  You may need to rerun ``cmake`` and
+    ``make`` after making changes to your code for everything to
+    build correctly.  Try running ``ccmake /path/to/geojs`` for a full
+    list of configuration options.
+
 Selenium testing
 ----------------
 
+The selenium testing infrastructure of Geojs is run via CTest, it assumes
+that the testing "server" is started prior to execution.  To start the
+server, just run ::
+
+    npm run start-test
+
+This will start a server on the default port of ``30100``.  The port
+and selenium host names are configurable with cmake.  For example inside
+the Kitware firewall, you can run the following to test on the selenium
+node on ``garant`` ::
+
+    cmake -DSELENIUM_TESTS=ON -DSELENIUM_HOST=garant /path/to/geojs
+    make
+    ctest -VV
+
+You may need to also set the variable ``TESTING_HOST`` to your computer's
+IP address reachable by the selenium node.
+
 Most tests for geojs require a full browser with webgl support.
 For these test, a framework based on `Selenium <http://docs.seleniumhq.org/>`_
 is provided.  This test framework is intentionally lightweight to allow
@@ -206,11 +255,3 @@ before running this script.  Note: you must have write permission in the MIDAS
 GeoJS community before you can upload new images.  Contact a community administrator
 for an invitation.
 
-Code coverage
--------------
-
-Code coverage information is generated automatically for all headless unit tests
-by Karma's test runner when running ``npm run test``.  The coverage information is
-submitted to `codecov <https://codecov.io/github/OpenGeoscience/geojs>`_ and
-`cdash <http://my.cdash.org/index.php?project=geojs>`_ after every
-successful Travis run.

docs/index.rst

@@ -25,6 +25,7 @@ If you have any questions or comments, feel free to join us on our
     quickstart
     users
     developers
+    provisioning
     testingutils
 
 Indices and tables

docs/provisioning.rst

@@ -0,0 +1,76 @@
+============================
+Provisioning for Development
+============================
+
+.. _ubuntu-development:
+
+Ubuntu 14.04
+-------------
+
+This shows how to set up a build and test environment in Ubuntu 14.04, using
+all but the Selenium-based tests.
+
+These instructions will probably work for any Ubuntu release from 14.04
+onward.  They assume a basic installation, as, for instance, from the
+`HashiCorp ubuntu/trusty64 image <https://atlas.hashicorp.com/ubuntu/boxes/trusty64>`_.
+
+Add nodejs to the sources so it can be installed ::
+
+    wget -qO- https://deb.nodesource.com/setup_4.x | sudo bash -
+
+Install required packages (you may want to also include cmake-curses-gui for
+convenience in configuring CMake options) ::
+
+    sudo apt-get install --yes \
+        cmake \
+        firefox \
+        git \
+        libjpeg8-dev \
+        libpango1.0-dev \
+        mesa-utils \
+        nodejs \
+        python-pip \
+        xvfb
+
+Install node-gyp, which is required to build the node canvas module ::
+
+    sudo npm install -g node-gyp-install && /usr/lib/node_modules/node-gyp-install/bin.js
+
+Checkout the GeoJS source and change to the source directory ::
+
+    git clone https://github.com/OpenGeoscience/geojs.git
+    cd geojs
+
+Install node modules ::
+
+    npm install
+
+Build GeoJS and run some basic tests ::
+
+    npm run build
+    npm run lint
+    npm run test
+
+Note that some of the tests measure speed, and therefore may fail if you are
+running on slow hardware or in a limited virtual machine.
+
+Use CMake to create additional tests and make to download test data ::
+
+    cmake .
+    make
+
+Run the headless WebGL tests ::
+
+    xvfb-run -s '-ac -screen 0 1280x1024x24' ctest -VV -R ffheadless
+
+Run all but the Selenium tests ::
+
+    xvfb-run -s '-ac -screen 0 1280x1024x24' ctest --output-on-failure -E selenium
+
+Install python packages ::
+
+    pip install --user girder-client
+
+Generate new baseline images for the headless WebGL tests ::
+
+    python test/baseline_images.py --xvfb --generate --upload --verbose

docs/quickstart.rst

@@ -12,16 +12,29 @@ The following software is required to build geojs from source:
 * `Git <http://git-scm.com/>`_
 * `Node.js <http://nodejs.org/>`_
 
-In addition, the following python modules are recommended for development
-and testing of geojs.
+For testing and development, the following additional software is required:
 
 * `Python 2.7 <http://www.python.org/>`_
-* `Make <http://www.gnu.org/software/make/>`_
 * `CMake <http://www.cmake.org/>`_
-* `Pillow <http://pillow.readthedocs.org/en/latest/>`_
+
+In addition, the following python modules are recommended for development
+and testing of geojs.
+
+* `Girder Client <http://girder.readthedocs.io>`_
+* `Pillow <http://pillow.readthedocs.io>`_
 * `Requests <http://docs.python-requests.org/en/latest/>`_
 * `Selenium <http://docs.seleniumhq.org/>`_
 
+For testing WebGL in a headless environment, the additional packages are needed:
+
+* `mesa-utils <http://www.mesa3d.org/>`_ and `libosmesa6 <http://www.mesa3d.org/>`_
+* `xvfb <https://www.x.org/archive/X11R7.6/doc/man/man1/Xvfb.1.xhtml>`_
+* `Firefox <http://www.mozilla.org/firefox>`_
+
+For an example on how to install all packages for a specific OS, see
+:ref:`Ubuntu 14.04 Provisioning <ubuntu-development>`.
+
+
 Getting the source code
 -----------------------