Merge pull request #1075 from numbbo/development

Merge of development into master before release

README.md

@@ -464,6 +464,23 @@ Details
   Jenkins on one ubuntu 12.04 machine, one OSX 10.9 machine, and one 32-bit
   Windows 7 machine with cygwin.
 
+Citation
+--------
+You may cite this work in a scientific context as
+
+N. Hansen, A. Auger, O. Mersmann, T. Tušar, D. Brockhoff. [COCO: A Platform for Comparing Continuous Optimizers in a Black-Box Setting](http://numbbo.github.io/coco-doc/), _ArXiv e-prints_, [arXiv:1603.08785](http://arxiv.org/abs/1603.08785), 2016.
+
+```
+@ARTICLE{hansen2016cocoplat, 
+  author = {Hansen, N. and Auger, A. and Mersmann, O. and 
+             Tu{\v s}ar, T. and Brockhoff, D.},
+  title = {{COCO}: A Platform for Comparing Continuous Optimizers 
+             in a Black-Box Setting},
+  journal = {ArXiv e-prints},
+  volume = {arXiv:1603.08785},
+  year = 2016
+}
+```
 
 Links and Documentation <a name="Links"></a>
 -----------------------

code-experiments/src/coco_string.c

@@ -446,11 +446,11 @@ static char *coco_string_trim(char *string) {
 	endp = string + len;
 
 	/* Move the front and back pointers to address the first non-whitespace characters from each end. */
-	while (isspace(*frontp)) {
+	while (isspace((unsigned char) *frontp)) {
 		++frontp;
 	}
 	if (endp != frontp) {
-		while (isspace(*(--endp)) && endp != frontp) {
+		while (isspace((unsigned char) *(--endp)) && endp != frontp) {
 		}
 	}
 

code-experiments/src/logger_biobj.c

@@ -18,8 +18,8 @@
  * for every nondominated solution. Whether these files are created, at what point in time the logger writes
  * nondominated solutions to the archive and whether the decision variables are output or not depends on
  * the values of log_nondom_mode and log_nondom_mode. See the bi-objective observer constructor
- * observer_biobj() for more information. One .adat file is created for each problem function and dimension
- * and contains information for all instances of that problem.
+ * observer_biobj() for more information. One .adat file is created for each problem function, dimension
+ * and instance.
  *
  * @note Whenever in this file a ROI is mentioned, it means the region of interest in the objective space.
  * The ROI is a rectangle with the ideal and nadir points as its two opposite vertices.
@@ -519,7 +519,7 @@ static logger_biobj_indicator_t *logger_biobj_indicator(const logger_biobj_data_
   		problem->problem_name);
   fprintf(indicator->tdat_file, "%% instance = %lu, reference value = %.*e\n",
   		(unsigned long) problem->suite_dep_instance, logger->precision_f, indicator->best_value);
-  fprintf(indicator->tdat_file, "%% function eval_number | indicator value\n");
+  fprintf(indicator->tdat_file, "%% function evaluation | indicator value\n");
 
   return indicator;
 }
@@ -762,7 +762,7 @@ static coco_problem_t *logger_biobj(coco_observer_t *observer, coco_problem_t *i
   logger_biobj_data_t *logger_biobj;
   observer_biobj_data_t *observer_biobj;
   const char nondom_folder_name[] = "archive";
-  char *path_name, *file_name = NULL, *prefix;
+  char *path_name, *file_name = NULL;
   size_t i;
 
   if (inner_problem->number_of_objectives != 2) {
@@ -801,15 +801,13 @@ static coco_problem_t *logger_biobj(coco_observer_t *observer, coco_problem_t *i
     coco_create_directory(path_name);
 
     /* Construct file name */
-    prefix = coco_remove_from_string(inner_problem->problem_id, "_i", "_d");
     if (logger_biobj->log_nondom_mode == LOG_NONDOM_ALL)
-      file_name = coco_strdupf("%s_nondom_all.adat", prefix);
+      file_name = coco_strdupf("%s_nondom_all.adat", inner_problem->problem_id);
     else if (logger_biobj->log_nondom_mode == LOG_NONDOM_FINAL)
-      file_name = coco_strdupf("%s_nondom_final.adat", prefix);
+      file_name = coco_strdupf("%s_nondom_final.adat", inner_problem->problem_id);
     coco_join_path(path_name, COCO_PATH_MAX, file_name, NULL);
     if (logger_biobj->log_nondom_mode != LOG_NONDOM_NONE)
       coco_free_memory(file_name);
-    coco_free_memory(prefix);
 
     /* Open and initialize the archive file */
     logger_biobj->adat_file = fopen(path_name, "a");

code-postprocessing/bbob_pproc/__main__.py

@@ -13,12 +13,15 @@
 import shutil
 import subprocess
 import doctest
+
 try:
     from . import rungeneric
+
     is_module = True
 except:
     is_module = False
 import matplotlib  # just to make sure the following is actually done first
+
 matplotlib.use('Agg')  # To avoid window popup and use without X forwarding
 
 # depreciated, to be removed, see end of file
@@ -48,10 +51,12 @@
     else:
         cocopp.rungeneric.main(args)
 
+
 def join_path(a, *p):
     path = os.path.join(a, *p)
     return path
 
+
 def copy_latex_templates():
     currentFolder = os.path.dirname(os.path.realpath(__file__))
     templateFolder = os.path.abspath(join_path(currentFolder, '..', 'latex-templates'))
@@ -60,6 +65,7 @@ def copy_latex_templates():
     shutil.copy(join_path(templateFolder, 'templateBBOBcmp.tex'), currentFolder)
     shutil.copy(join_path(templateFolder, 'templateBBOBmany.tex'), currentFolder)
     shutil.copy(join_path(templateFolder, 'templateBIOBJarticle.tex'), currentFolder)
+    shutil.copy(join_path(templateFolder, 'templateBIOBJmultiple.tex'), currentFolder)
     shutil.copy(join_path(templateFolder, 'sig-alternate.cls'), currentFolder)
     shutil.copy(join_path(templateFolder, 'comment.sty'), currentFolder)
     shutil.copy(join_path(templateFolder, 'acmcopyright.sty'), currentFolder)
@@ -87,6 +93,7 @@ def retrieve_algorithm(dataPath, folderName, algorithmName, fileName=None):
         dataurl = 'http://coco.gforge.inria.fr/data-archive/%s/%s' % (folderName, algorithmName)
         urllib.urlretrieve(dataurl, algorithmFile)
 
+
 def prepare_data(run_all_tests):
     print('preparing algorithm data')
 
@@ -107,9 +114,12 @@ def prepare_data(run_all_tests):
         retrieve_algorithm(dataPath, '2013', 'lmm-CMA-ES_auger_noiseless.tgz')
         retrieve_algorithm(dataPath, '2009', 'DE-PSO_garcia-nieto_noiseless.tgz')
         retrieve_algorithm(dataPath, '2009', 'VNS_garcia-martinez_noiseless.tgz')
+        retrieve_algorithm(dataPath, 'biobj-test', 'RS-4.tgz')
+        retrieve_algorithm(dataPath, 'biobj-test', 'RS-100.tgz')
 
     return dataPath
 
+
 def process_doctest_output(stream=None):
     """ """
     import fileinput
@@ -128,7 +138,7 @@ def process_doctest_output(stream=None):
 
         if state == -1:  # found a failed example line
             s1 += '\n\n*** Failed Example:' + line
-            s2 += '\n\n\n'   # line
+            s2 += '\n\n\n'  # line
             # state = 0  # wait for 'Expected:' line
 
         if line.startswith('Expected:'):
@@ -150,6 +160,11 @@ def process_doctest_output(stream=None):
         if state == 2:
             s2 += line + ''
 
+
+def delete_files():
+    shutil.rmtree('ppdata')
+
+
 def main(args):
     """these tests are executed when ``python bbob_pproc`` is called.
 
@@ -162,7 +177,7 @@ def main(args):
 
     python = 'python -m '  # how to call python
     if len(sys.argv) > 1 and sys.argv[1] == 'wine':
-        python = 'C:\\Python26\\python.exe ' # works for wine
+        python = 'C:\\Python26\\python.exe '  # works for wine
 
     data_path = ' ' + prepare_data(run_all_tests)
 
@@ -178,57 +193,82 @@ def main(args):
     print('**  subtest 1 finished in ', time.time() - t0, ' seconds')
     assert result == 0, 'Test failed: rungeneric on one algorithm with option --conv.'
     run_latex_template("templateBBOBarticle.tex")
+    delete_files()
 
     t0 = time.time()
     print(python + command + join_path(data_path, 'RS.tgz'))
     result = os.system(python + command + join_path(data_path, 'RS.tgz'))
-    print('**  subtest 1 finished in ', time.time() - t0, ' seconds')
+    print('**  subtest 2 finished in ', time.time() - t0, ' seconds')
     assert result == 0, 'Test failed: rungeneric on one bi-objective algorithm.'
     run_latex_template("templateBIOBJarticle.tex")
+    delete_files()
 
     if run_all_tests:
         t0 = time.time()
         print(time.asctime())
-        result = os.system(python + command + # ' --omit-single ' +
-                    join_path(data_path, 'BIPOP-CMA-ES_hansen_noiseless.tgz') +
-                    join_path(data_path, 'MCS_huyer_noiseless.tgz') +
-                    join_path(data_path, 'NEWUOA_ros_noiseless.tgz') +
-                    join_path(data_path, 'RANDOMSEARCH_auger_noiseless.tgz') +
-                    join_path(data_path, 'BFGS_ros_noiseless.tgz'))
-        print('**  subtest 2 finished in ', time.time() - t0, ' seconds')
+        result = os.system(python + command + ' --omit-single ' +
+                           join_path(data_path, 'BIPOP-CMA-ES_hansen_noiseless.tgz') +
+                           join_path(data_path, 'MCS_huyer_noiseless.tgz') +
+                           join_path(data_path, 'NEWUOA_ros_noiseless.tgz') +
+                           join_path(data_path, 'RANDOMSEARCH_auger_noiseless.tgz') +
+                           join_path(data_path, 'BFGS_ros_noiseless.tgz'))
+        print('**  subtest 3 finished in ', time.time() - t0, ' seconds')
         assert result == 0, 'Test failed: rungeneric on many algorithms.'
         run_latex_template("templateBBOBmany.tex")
+        delete_files()
 
         t0 = time.time()
         result = os.system(python + command + '--conv' +
-                    join_path(data_path, 'SMAC-BBOB_hutter_noiseless.tgz') +
-                    join_path(data_path, 'lmm-CMA-ES_auger_noiseless.tgz'))
-        print('**  subtest 3 finished in ', time.time() - t0, ' seconds')
+                           join_path(data_path, 'SMAC-BBOB_hutter_noiseless.tgz') +
+                           join_path(data_path, 'lmm-CMA-ES_auger_noiseless.tgz'))
+        print('**  subtest 4 finished in ', time.time() - t0, ' seconds')
         assert result == 0, 'Test failed: rungeneric on two algorithms with option --conv.'
         run_latex_template("templateBBOBcmp.tex")
+        delete_files()
 
         t0 = time.time()
         result = os.system(python + command + ' --omit-single ' +
-                    join_path(data_path, 'DE-PSO_garcia-nieto_noiseless.tgz') +
-                    join_path(data_path, 'VNS_garcia-martinez_noiseless.tgz'))
-        print('**  subtest 4 finished in ', time.time() - t0, ' seconds')
+                           join_path(data_path, 'DE-PSO_garcia-nieto_noiseless.tgz') +
+                           join_path(data_path, 'VNS_garcia-martinez_noiseless.tgz'))
+        print('**  subtest 5 finished in ', time.time() - t0, ' seconds')
         assert result == 0, 'Test failed: rungeneric on two algorithms with option --omit-single.'
         run_latex_template("templateBBOBcmp.tex")
+        delete_files()
 
         t0 = time.time()
         result = os.system(python + command + ' --expensive ' +
-                    join_path(data_path, 'VNS_garcia-martinez_noiseless.tgz'))
-        print('**  subtest 5 finished in ', time.time() - t0, ' seconds')
+                           join_path(data_path, 'VNS_garcia-martinez_noiseless.tgz'))
+        print('**  subtest 6 finished in ', time.time() - t0, ' seconds')
         assert result == 0, 'Test failed: rungeneric on one algorithm with option --expensive.'
         run_latex_template("templateBBOBarticle.tex")
+        delete_files()
+
+        t0 = time.time()
+        result = os.system(python + command + ' --omit-single ' +
+                           join_path(data_path, 'RS.tgz') +
+                           join_path(data_path, 'RS-4.tgz'))
+        print('**  subtest 7 finished in ', time.time() - t0, ' seconds')
+        assert result == 0, 'Test failed: rungeneric on two bi-objective algorithms.'
+        run_latex_template("templateBIOBJmultiple.tex")
+        delete_files()
+
+        t0 = time.time()
+        result = os.system(python + command + ' --omit-single ' +
+                           join_path(data_path, 'RS.tgz') +
+                           join_path(data_path, 'RS-4.tgz') + 
+                           join_path(data_path, 'RS-100.tgz'))
+        print('**  subtest 8 finished in ', time.time() - t0, ' seconds')
+        assert result == 0, 'Test failed: rungeneric on three bi-objective algorithms.'
+        run_latex_template("templateBIOBJmultiple.tex")
+        delete_files()
 
     print('launching doctest (it might be necessary to close a few pop up windows to finish)')
     t0 = time.time()
 
     if 1 < 3:
         failure_count = 0
         test_count = 0
-        #doctest.testmod(report=True, verbose=True)  # this is quite cool!
+        # doctest.testmod(report=True, verbose=True)  # this is quite cool!
         # go through the py files in the bbob_pproc folder
         currentPath = os.getcwd()
         newPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
@@ -259,6 +299,7 @@ def main(args):
     if failure_count > 0:
         raise ValueError('%d of %d tests failed' % (failure_count, test_count))
 
+
 """
         sys.path.append(os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
         import bbob_pproc as bb

code-postprocessing/bbob_pproc/bestalg.py

@@ -18,6 +18,7 @@
 """
 
 from __future__ import absolute_import
+from __future__ import print_function
 
 import os
 import sys
@@ -349,7 +350,7 @@ def load_best_algorithm(force=False):
         bestAlgorithmEntries = None
         return bestAlgorithmEntries
 
-    print("Loading best algorithm data from %s ..." % bestAlgorithmFilename)
+    print("  Loading best algorithm data from %s ..." % bestAlgorithmFilename)
     sys.stdout.flush()
 
     bestAlgFilePath = os.path.split(__file__)[0]
@@ -366,7 +367,8 @@ def load_best_algorithm(force=False):
         bestAlgorithmEntries = None
 
     fid.close()
-    print_done()
+    #print("  ", end="")
+    #print_done()
 
     return bestAlgorithmEntries
 

code-postprocessing/bbob_pproc/cococommands.py

@@ -51,12 +51,8 @@
 
 from __future__ import absolute_import
 
-#from bbob_pproc import ppsingle, ppfigdim, dataoutput
-# from bbob_pproc.pproc import DataSetList, DataSet
 from . import pproc
 
-#__all__ = ['load', 'info', 'pickle', 'systeminfo', 'DataSetList', 'DataSet']
-
 def load(filename):
     """Create a :py:class:`DataSetList` instance from a file or folder.
 

code-postprocessing/bbob_pproc/comp2/pptable2.py

@@ -440,7 +440,7 @@ def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True):
         res = ("").join(str(item) for item in tableHtml)
         res = '<p><b>%d-D</b></p>\n<table>\n%s</table>\n' % (d, res)
 
-        filename = os.path.join(outputdir, genericsettings.two_algorithm_file_name + '.html')
+        filename = os.path.join(outputdir, genericsettings.pptable2_file_name + '.html')
         lines = []
         with open(filename) as infile:
             for line in infile:

code-postprocessing/bbob_pproc/compall/ppfigs.py

@@ -145,7 +145,7 @@ def prepare_ecdfs_figure_caption():
 def ecdfs_figure_caption(for_html = False, dimension = 0):
 
     if for_html:
-        key = '##bbobECDFslegend%s%d##' % (testbedsettings.current_testbed.scenario, dimension)
+        key = '##bbobECDFslegend%s##' % testbedsettings.current_testbed.scenario
         caption = htmldesc.getValue(key)
     else:
         caption = prepare_ecdfs_figure_caption()
@@ -156,6 +156,7 @@ def ecdfs_figure_caption(for_html = False, dimension = 0):
 
     caption = caption.replace('BBOBPPFIGSTARGETRANGE',
                               str(testbedsettings.current_testbed.pprldmany_target_range_latex))
+    caption = caption.replace('DIMVALUE', str(dimension))
 
     if genericsettings.runlength_based_targets:
         caption = caption.replace('REFERENCE_ALGORITHM', target.reference_algorithm)
@@ -571,8 +572,6 @@ def main(dictAlg, htmlFilePrefix, isBiobjective, sortedAlgs=None, outputdir='ppd
                 )
 
         toolsdivers.replace_in_file(htmlFile, '##bbobppfigslegend##', scaling_figure_caption(True) + 'Legend: ' + alg_definitions_html)
-        toolsdivers.replace_in_file(htmlFile, '##bbobECDFslegend5##', ecdfs_figure_caption(True, 5))
-        toolsdivers.replace_in_file(htmlFile, '##bbobECDFslegend20##', ecdfs_figure_caption(True, 20))
 
         if verbose:
             print 'Wrote commands and legend to %s' % filename