Fix benchmarker csv format, time and memory results output at same time + Emscripten O2

test/wasmBenchmarker/benchmark.py

@@ -21,6 +21,7 @@
 import sys
 import time
 
+from pathlib import Path
 from os.path import abspath, dirname, join
 from markdownTable import markdownTable  # pip install py-markdown-table
 
@@ -30,31 +31,31 @@
 DIFF_TRESHOLD = 1e-7
 
 expectedValues = {
-    "change": 4,
-    "factorial": 30,
-    "fannkuch": 120,
-    "fibonacci": 1,
+    "change": 3,
+    "factorial": 899999994000000000,
+    "fannkuch": 360,
+    "fibonacci": 63245986,
     "gregory": 3.14159264,
-    "hanoi": 0,
+    "hanoi": 67108863,
     "heapsort": 0,
     "huffman": 0,
     "kNucleotide": 1,
-    "mandelbrotFloat": 775007,
-    "mandelbrotDouble": 775007,
+    "mandelbrotFloat": 775014,
+    "mandelbrotDouble": 775014,
     "matrixMultiply": 3920.0,
     "miniWalrus": 27449,
-    "nbody": -0.16910574,
-    "nqueens": 0,
-    "prime": 48611,
+    "nbody": -0.16904405,
+    "nqueens": 246,
+    "prime": 70657,
     "quickSort": 0,
-    "redBlack": 4000000,
+    "redBlack": 13354000,
     "rsa": 0,
-    "salesman": 840,
-    "simdMandelbrotFloat": 775007,
-    "simdMandelbrotDouble": 775007,
-    "simdNbody": -0.16910574,
+    "salesman": 2520,
+    "simdMandelbrotFloat": 775014,
+    "simdMandelbrotDouble": 775014,
+    "simdNbody": -0.16904405,
     "simdMatrixMultiply": 3920.0,
-    "ticTacToe": 4748900
+    "ticTacToe": 18995600
 }
 
 # https://benchmarksgame-team.pages.debian.net/benchmarksgame/description/simple.html#simple
@@ -79,6 +80,7 @@ def parse_args():
     parser.add_argument("--mem", help="measure MAX RSS", action="store_true")
     parser.add_argument("--jit", help="use JIT version of Walrus", action="store_true")
     parser.add_argument("--verbose", help="prints extra informations e.g. paths", action="store_true")
+    parser.add_argument("--no-system-emcc", help="Don't use emcc command from the system", action="store_true", default=False)
     return parser.parse_args()
 
 
@@ -94,8 +96,14 @@ def check_programs(engines, verbose):
     if (verbose): print("Checks done")
 
 
-def get_emcc(verbose):
+def get_emcc(verbose, system_emcc=True):
     emcc_path = None 
+
+    if system_emcc and os.system("emcc --version >/dev/null") == 0:
+        if (verbose): print("Emscripten already installed on system")
+        emcc_path = "emcc"
+        return emcc_path
+
     if os.getenv("EMSDK"):
         emcc_path = join(os.getenv("EMSDK"), "upstream/emscripten/emcc.py")
         if os.path.exists(emcc_path):
@@ -105,20 +113,19 @@ def get_emcc(verbose):
     if os.path.exists("./emsdk/.git"):
         os.system("(cd ./emsdk && git fetch -a) >/dev/null")
         os.system("(cd ./emsdk && git reset --hard origin/HEAD) >/dev/null")
-        os.system("./emsdk/emsdk install latest >/dev/null")
-        os.system("./emsdk/emsdk activate latest >/dev/null")
     else:
         os.system("git clone --depth 1 https://github.com/emscripten-core/emsdk.git ./emsdk >/dev/null")
-        os.system("./emsdk/emsdk install latest >/dev/null")
-        os.system("./emsdk/emsdk activate latest >/dev/null")
+
+    os.system("./emsdk/emsdk install latest >/dev/null")
+    os.system("./emsdk/emsdk activate latest >/dev/null")
 
     emcc_path = "./emsdk/upstream/emscripten/emcc"
     if (verbose): print(f"EMCC install done: {emcc_path}")
     return emcc_path
 
 
 def compile_tests(emcc_path, path, only_game, only_simd, compile_anyway, run, verbose):
-    if not os.path.exists(emcc_path):
+    if os.system(f"{emcc_path} --version >/dev/null") != 0:
         raise Exception(f"Invalid path for emcc: {emcc_path}")
 
     if not os.path.exists(path):
@@ -149,6 +156,7 @@ def compile_tests(emcc_path, path, only_game, only_simd, compile_anyway, run, ve
         flags = "-msimd128" if file.startswith("simd") else ""
         flags += (" -s WASM=1 -s EXPORTED_FUNCTIONS=_runtime"
                   " -s EXPORTED_RUNTIME_METHODS=ccall,cwrap"
+                  " -O2"
                   f" -o {path}/wasm/{name}.wasm")
         if (verbose): print(f"compiling {name}")
         command = f"{emcc_path} {path}/{file} {flags}"
@@ -242,11 +250,14 @@ def generate_report(data, file_name=None):
                 engine_names.remove("test")
             for engineName in engine_names:
                 header += f";{engineName}"
+            header += "\n"
             file.write(header)
             for record in data:
                 line = record["test"]
                 for engineName in engine_names:
                     line += f";{record[engineName]}"
+
+                line += "\n"
                 file.write(line)
             return
         file.write(markdownTable(data).setParams(row_sep="markdown", quote=False).getMarkdown())
@@ -260,14 +271,20 @@ def main():
         print("You need to specify the engine locations", file=sys.stderr)
         exit(1)
 
+    memreport = None
+    if args.report is not None:
+        memreport = Path(args.report).absolute()
+        memreport = memreport.parent / f"{memreport.stem}_mem{memreport.suffix}"
+        memreport = str(memreport)
+
     check_programs(args.engines, args.verbose)
-    emcc_path = get_emcc(args.verbose)
+    emcc_path = get_emcc(args.verbose, not args.no_system_emcc)
     test_names = compile_tests(emcc_path, args.test_dir, args.only_game, args.only_simd, args.compile_anyway, args.run, args.verbose)
 
     result_data = run_tests(args.test_dir, test_names, args.engines, args.iterations, args.mem, args.jit, args.verbose)
     generate_report(result_data["time"], args.report)
     if (args.mem):
-        generate_report(result_data["mem"], args.report)
+        generate_report(result_data["mem"], memreport)
 
 
 if __name__ == "__main__":

test/wasmBenchmarker/ctests/change.c

@@ -20,7 +20,7 @@
 uint32_t coins[] = {5, 10, 20, 50, 100, 200};
 
 #define COINT_NUMBER sizeof(coins) / sizeof(uint32_t)
-#define MONEY 85
+#define MONEY 155
 
 /*
  * Return the smallest number of coins

test/wasmBenchmarker/ctests/factorial.c

@@ -17,9 +17,9 @@
 #include <stdint.h>
 #include <stdio.h>
 
-uint64_t factorial(uint8_t n) {
-    uint8_t counter = 0;
-    for (uint8_t i = 0; i < n; i++, counter++) {
+uint64_t factorial(uint64_t n) {
+    uint64_t counter = 0;
+    for (uint64_t i = 0; i < n; i++, counter++) {
         factorial(n - 1);
     }
 
@@ -29,8 +29,10 @@ uint64_t factorial(uint8_t n) {
 uint64_t runtime() {
     uint64_t retVal = 0;
 
-    for (uint16_t i = 0; i < 3; i++) {
-        retVal += factorial(10);
+    for (uint64_t i = 0; i < 6000000000; i++) {
+        retVal += factorial(150000000);
+        retVal -= factorial(149999999);
+        retVal += factorial(149999998);
     }
 
     return retVal;

test/wasmBenchmarker/ctests/fannkuch.c

@@ -78,7 +78,7 @@ uint64_t fannkuch(uint64_t n) {
 uint64_t runtime() {
     uint64_t retVal = 0;
 
-    for (uint8_t i = 0; i < 4; i++) {
+    for (uint8_t i = 0; i < 12; i++) {
         retVal += fannkuch(9);
     }
 

test/wasmBenchmarker/ctests/fibonacci.c

@@ -19,17 +19,15 @@
 
 uint64_t fibonacci(uint64_t n) {
     if (n == 0 || n == 1) {
-        return 1;
+        return n;
     }
 
-    fibonacci(n - 1);
-    fibonacci(n - 2);
 
-    return fibonacci(n - 2);
+    return fibonacci(n - 1) + fibonacci(n - 2);
 }
 
 uint64_t runtime() {
-    return fibonacci(26);
+    return fibonacci(39);
 }
 
 int main() {

test/wasmBenchmarker/ctests/gregory.c

@@ -28,7 +28,7 @@ double gregorySeries(uint64_t n) {
 }
 
 double runtime() {
-    return gregorySeries(75000000);
+    return gregorySeries(300000000);
 }
 
 int main() {

test/wasmBenchmarker/ctests/hanoi.c

@@ -20,19 +20,21 @@
 #define EXIT_SUCCESS 0
 #define EXIT_FAILURE 1
 
-#define DISKS 24
+#define DISKS 26
 
 typedef struct {
     uint64_t disks[DISKS];
     uint64_t size; // current size
 } rod;
 
+uint32_t steps = 0;
 void move_disk(rod *from,
                rod *to) {
     if (from->size == 0) {
         return;
     }
     if (to->size == 0) {
+        steps++;
         to->disks[0] = from->disks[from->size - 1];
         from->size -= 1;
         to->size += 1;
@@ -41,6 +43,7 @@ void move_disk(rod *from,
     if (from->disks[from->size - 1] < to->disks[to->size - 1]) {
         return;
     }
+    steps++;
     to->disks[to->size] = from->disks[from->size - 1];
     from->size -= 1;
     to->size += 1;
@@ -64,6 +67,7 @@ void move_tower(rod *from,
 
 uint64_t hanoi() {
     // init
+    steps = 0;
     rod rods[3];
     for (int i = 0; i < DISKS; i++) {
         rods[0].disks[i] = i;
@@ -86,7 +90,7 @@ uint64_t hanoi() {
             return EXIT_FAILURE;
         }
     }
-    return EXIT_SUCCESS;
+    return steps;
 }
 
 uint64_t runtime() {

test/wasmBenchmarker/ctests/heapsort.c

@@ -19,7 +19,7 @@
 #include "include/random.h"
 
 #define ARRAY_LENGTH 15000
-#define ITERATIONS 64
+#define ITERATIONS 300
 
 #define NO_CHILD 0
 #define NO_PARENT 65535

test/wasmBenchmarker/ctests/huffman.c

@@ -29,7 +29,7 @@ byte runtime();
 
 // TEST PARAMETER
 
-#define ITERATIONS 3
+#define ITERATIONS 7
 
 char* message = "Lorem ipsum dolor sit amet, et wisi primis duo."
                 "In quo erat tritani fuisset, no ullum vivendo "

test/wasmBenchmarker/ctests/include/mandelbrot.h

@@ -34,8 +34,8 @@
 #include <math.h>
 #include <stdint.h>
 
-#define WIDTH 1600
-#define HIGHT 1400
+#define WIDTH 12800
+#define HIGHT 11200
 #define N 20
 #define REAL_AXIS_SHIFT -1.8 // ~ horizontal shift
 #define IMAGINARY_AXIS_SHIFT -1.0 // ~ vertical shift

test/wasmBenchmarker/ctests/include/simdMandelbrot.h

@@ -35,8 +35,8 @@
 
 #include <wasm_simd128.h>
 
-#define WIDTH 1600
-#define HIGHT 1400
+#define WIDTH 6400
+#define HIGHT 5600
 #define N 20
 #define REAL_AXIS_SHIFT -1.8 // ~ horizontal shift
 #define IMAGINARY_AXIS_SHIFT -1.0 // ~ vertical shift

test/wasmBenchmarker/ctests/kNucleotide.c

@@ -18,6 +18,8 @@
 #include <stdint.h>
 #include <stdbool.h>
 
+#define LOOP 2
+
 extern const char *input_3;
 
 typedef struct {
@@ -221,14 +223,26 @@ bool check() {
 }
 
 bool runtime() {
-    frequency(input_3, 1);
-    frequency(input_3, 2);
-    count(input_3, "ggt");
-    count(input_3, "ggta");
-    count(input_3, "ggtatt");
-    count(input_3, "ggtattttaatt");
-    count(input_3, "ggtattttaatttatagt");
-    return check();
+    uint8_t checks = 0;
+    for(uint8_t i = 0; i < LOOP; i++) {
+        for(uint8_t j = 0; j < record_current_size; j++) {
+            for (uint8_t k = 0; k < 50; k++) {
+                records[j].key[k] = 0;
+            }
+            records[j].value=0;
+        }
+        frequency(input_3, 1);
+        frequency(input_3, 2);
+        count(input_3, "ggt");
+        count(input_3, "ggta");
+        count(input_3, "ggtatt");
+        count(input_3, "ggtattttaatt");
+        count(input_3, "ggtattttaatttatagt");
+        if (check()) {
+            checks += 1;
+        }
+    }
+    return checks == LOOP;
 }
 
 const char *input_3 =

test/wasmBenchmarker/ctests/matrixMultiply.c

@@ -18,8 +18,8 @@
 #include <stdio.h>
 #include <stdint.h>
 
-// 4x4 square matrix
-#define MATRIX_SIZE 16
+// 13x13 square matrix
+#define MATRIX_SIZE 169
 
 #define ITERATION 3500000
 

test/wasmBenchmarker/ctests/nbody.c

@@ -17,7 +17,7 @@
 #include <stdio.h>
 #include <math.h>
 
-#define LOOP 450000
+#define LOOP 3000000
 #define SOLAR_MASS 39.47841760435743f
 #define DAYS_PER_YEAR 365.24f
 #define BODIES_COUNT 5