Merge pull request #802 from google/embench_tflm_dse_updates

Embench tflm dse updates

common/src/models/mlcommons_tiny_v01/README.md

@@ -19,7 +19,7 @@ not implemented in the tests -- rather a 32-bit xor reduction is used to check
 for output equivalence to expected results.
 
 > To include this model in your project, define the identifier
-> `INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD`.
+> `INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD`.
 
 ### Image Classification
 
@@ -30,7 +30,7 @@ Inputs to the tests have already been quantized, and are located in
 `imgc/test_data`.
 
 > To include this model in your project, define the identifier
-> `INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC`.
+> `INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC`.
 
 ### Keyword Spotting
 
@@ -41,7 +41,7 @@ Inputs to the tests do not need to be quantized and are located in
 `kws/test_data`.
 
 > To include this model in your project, define the identifier
-> `INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS`.
+> `INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS`.
 
 ### Visual Wake Words
 
@@ -52,4 +52,4 @@ Inputs to the tests have already been quantized, and are located in
 `vww/test_data`.
 
 > To include this model in your project, define the identifier
-> `INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW`.
+> `INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW`.

common/src/models/models.c

@@ -55,19 +55,19 @@ static struct Menu MENU = {
 #if defined(INCLUDE_MODEL_HPS)
         MENU_ITEM(AUTO_INC_CHAR, "HPS models", hps_model_menu),
 #endif
-#if defined(INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD)
+#if defined(INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD)
         MENU_ITEM(AUTO_INC_CHAR, "MLCommons Tiny V0.1 Anomaly Detection",
                   mlcommons_tiny_v01_anomd_menu),
 #endif
-#if defined(INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC)
+#if defined(INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC)
         MENU_ITEM(AUTO_INC_CHAR, "MLCommons Tiny V0.1 Image Classification",
                   mlcommons_tiny_v01_imgc_menu),
 #endif
-#if defined(INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS)
+#if defined(INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS)
         MENU_ITEM(AUTO_INC_CHAR, "MLCommons Tiny V0.1 Keyword Spotting",
                   mlcommons_tiny_v01_kws_menu),
 #endif
-#if defined(INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW)
+#if defined(INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW)
         MENU_ITEM(AUTO_INC_CHAR, "MLCommons Tiny V0.1 Visual Wake Words",
                   mlcommons_tiny_v01_vww_menu),
 #endif

common/src/tflite.cc

@@ -95,16 +95,16 @@ constexpr int kTensorArenaSize = const_max<int>(
 #ifdef INCLUDE_MODEL_HPS
     256 * 1024,
 #endif
-#ifdef INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#ifdef INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
     3 * 1024,
 #endif
-#ifdef INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+#ifdef INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
     53 * 1024,
 #endif
-#ifdef INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#ifdef INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
     23 * 1024,
 #endif
-#ifdef INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#ifdef INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
     99 * 1024,
 #endif
     0 /* When no models defined, we don't need a tensor arena. */

common/src/third_party/embench_iot_v1/src/embench_wrapper.c

@@ -26,10 +26,10 @@
 #define GET_TIMER()   perf_get_litex_timer()
 #define PRINT_PERF(a, b)  printf("Spent %llu cycles\n", a - b) 
 #else // default timer to use is mcycle
-#define TIMER_TYPE    unsigned int
-#define RESET_TIMER() /* no reset for mcycle */
-#define GET_TIMER()   perf_get_mcycle() 
-#define PRINT_PERF(a, b)  printf("Spent %u cycles\n", b - a) 
+#define TIMER_TYPE    uint64_t
+#define RESET_TIMER() perf_reset_all_counters() 
+#define GET_TIMER()   perf_get_mcycle64() 
+#define PRINT_PERF(a, b)  perf_print_all_counters(); perf_print_value(b - a); printf(" cycles total\n")
 #endif
 
 
@@ -45,7 +45,7 @@ void embench_wrapper_##benchname () \
   result = benchmark_##benchname (); \
   TIMER_TYPE end = GET_TIMER(); \
   correct = verify_benchmark_##benchname (result); \
-  printf("%s\n", correct ? "OK  Benchmark result verified" : "FAIL  Benchmark result incorrect"); \
+  printf("%s\n", correct ? "OK  Benchmark test passed" : "FAIL  Benchmark test failed"); \
   PRINT_PERF(start, end); \
 }
 

proj/dse_template/Makefile

@@ -31,14 +31,43 @@ DEFINES += INCLUDE_MODEL_PDTI8
 #DEFINES += INCLUDE_MODEL_MAGIC_WAND
 #DEFINES += INCLUDE_MODEL_MNV2
 #DEFINES += INCLUDE_MODEL_HPS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
 
 # Uncomment to include all TFLM examples (pdti8, micro_speech, magic_wand)
 #DEFINES += INCLUDE_ALL_TFLM_EXAMPLES
 
+# Uncomment to include specified Embench workload in built binary
+#DEFINES += INCLUDE_EMBENCH_PRIMECOUNT 
+#DEFINES += INCLUDE_EMBENCH_MINVER 
+#DEFINES += INCLUDE_EMBENCH_AHA_MONT64
+#DEFINES += INCLUDE_EMBENCH_CRC_32
+#DEFINES += INCLUDE_EMBENCH_CUBIC
+#DEFINES += INCLUDE_EMBENCH_EDN
+#DEFINES += INCLUDE_EMBENCH_HUFFBENCH
+#DEFINES += INCLUDE_EMBENCH_MATMUL
+#DEFINES += INCLUDE_EMBENCH_MD5
+#DEFINES += INCLUDE_EMBENCH_NBODY
+#DEFINES += INCLUDE_EMBENCH_NETTLE_AES
+#DEFINES += INCLUDE_EMBENCH_NETTLE_SHA256
+#DEFINES += INCLUDE_EMBENCH_NSICHNEU
+#DEFINES += INCLUDE_EMBENCH_PICOJPEG
+#DEFINES += INCLUDE_EMBENCH_QRDUINO
+#DEFINES += INCLUDE_EMBENCH_SLRE
+#DEFINES += INCLUDE_EMBENCH_ST
+#DEFINES += INCLUDE_EMBENCH_STATEMATE
+#DEFINES += INCLUDE_EMBENCH_TARFIND
+#DEFINES += INCLUDE_EMBENCH_UD
+#DEFINES += INCLUDE_EMBENCH_WIKISORT
+
+# Uncomment to include all Embench examples  
+#DEFINES += INCLUDE_ALL_EMBENCH_EXAMPLES
+
+# Uncomment to exclude TFLM src in build
+#export SKIP_TFLM=1
+
 # Uncomment to program menu sequence as you wish to run workload w/o interacting with menu 
 DEFINES += DSE_SIMULATION
 DEFINES += MENU_CHAR_SEQUENCE='"11gxxQ"'  # this default character sequence runs golden tests for PDIT8

proj/dse_template/dse_framework.py

@@ -66,7 +66,10 @@ def get_cycle_count():
 
     cycles = sys.float_info.max 
     cycles_found = False
+    test_passing = True
     for line in lines:
+        if 'FAIL' in line and 'test' in line and 'failed' in line:
+            test_passing = False
         if 'cycles total' in line:
             cycles_list = [float(s) for s in line.split() if s.isdigit()]
             cur_cycles = cycles_list[0] 
@@ -76,15 +79,17 @@ def get_cycle_count():
             else:
                 cycles = cur_cycles if cur_cycles > cycles else cycles
 
-    return cycles
+    return (test_passing, cycles)
 
 
-def run_config(variant, target):
+def run_config(variant, target="digilent_arty", workload="pdti8"):
 
     # Generate bitstream and run simulation for given CPU variant and return metric results
     cycles = None
     cells  = None
     run_succeeds = False 
+    test_passed = False
+    update_workload(workload)
     while not run_succeeds:
         EXTRA_LITEX_ARGS = 'EXTRA_LITEX_ARGS="--cpu-variant=' + variant +'"'
         subprocess.run(['make', 'clean']) 
@@ -94,7 +99,7 @@ def run_config(variant, target):
         outfile  = open(filename, "w")
         workload = subprocess.run(workload_cmd, stdout=outfile)
         try:
-            cycles = get_cycle_count()
+            test_passed, cycles = get_cycle_count()
             run_succeeds = True
         except UnicodeDecodeError:
             run_succeeds = False
@@ -103,16 +108,50 @@ def run_config(variant, target):
     # Remove any copied tf source overlay for next run
     if os.path.exists("./src/tensorflow"):
         subprocess.run(['rm', '-rf', './src/tensorflow'])
+
+    # Check to see if the workload / benchmark test still passing (i.e. hw and sw are both functionally correct)
+    if test_passed:
+        return (cycles, cells)
+    else: # if test failing return large number to make sample invalid
+        print("Simulation completed but program test failed! Modifications need to be made to CFU HW or SW.")
+        return (float('inf'), float('inf'))
+
+
+def update_workload(workload="pdti8", makefile_path="./Makefile"):
+    if not os.path.isfile(makefile_path):
+        print("Makefile not found.")
+        exit() 
 
-    return (cycles, cells)
+    with open(makefile_path, "r") as file:
+        lines = file.readlines()
+
+    new_lines = []
+    embench_workload = False
+    for line in lines:
+        if workload.lower() in line.lower():
+            embench_workload = "EMBENCH" in line
+            if "#DEFINES" in line:
+                line = line.replace("#DEFINES", "DEFINES") 
+        elif ("INCLUDE_MODEL_" in line or "INCLUDE_EMBENCH_" in line) and "#DEFINES" not in line:
+            line = line.replace("DEFINES", "#DEFINES") 
+        elif "MENU_CHAR_SEQUENCE" in line:
+            if embench_workload:
+                line = "DEFINES += MENU_CHAR_SEQUENCE='\"81xQ\"'\n"  
+            else: # TinyML model workload golden tests
+                line = "DEFINES += MENU_CHAR_SEQUENCE='\"11gxxQ\"'\n"  
+        new_lines.append(line)
+
+    with open(makefile_path, "w") as file:
+        file.writelines(new_lines)
 
 
 def dse(csrPluginConfig, bypass, cfu, dCacheSize, hardwareDiv, 
-        iCacheSize, mulDiv, prediction, safe, singleCycleShift, singleCycleMulDiv, target):
+        iCacheSize, mulDiv, prediction, safe, singleCycleShift, 
+        singleCycleMulDiv, target="digilent_arty", workload="pdti8"):
 
     variant = make_variant_string(csrPluginConfig, bypass, cfu, dCacheSize, hardwareDiv, 
         iCacheSize, mulDiv, prediction, safe, singleCycleShift, singleCycleMulDiv)
-    cycles, cells = run_config(variant, target)
+    cycles, cells = run_config(variant, target, workload)
 
     print("NUMBER OF CYCLES: "  + str(cycles))
     print("NUMBER OF CELLS:  "  + str(cells))
@@ -123,7 +162,7 @@ def dse(csrPluginConfig, bypass, cfu, dCacheSize, hardwareDiv,
 
 if __name__ == "__main__":
     if len(sys.argv) > 1 :
-        # Used for running dse vis cmd line
+        # Used for running dse via cmd line
         # Sample command that would be equivalent to default params:
         # ./dse_framework.py mcycle True False 4096 True 4096 True none True True True 
         csrPluginConfig   = sys.argv[1]
@@ -138,6 +177,7 @@ def dse(csrPluginConfig, bypass, cfu, dCacheSize, hardwareDiv,
         singleCycleShift  = True if sys.argv[10] == "True" else False
         singleCycleMulDiv = True if sys.argv[11] == "True" else False
         TARGET            = "digilent_arty"  
+        WORKLOAD          = "pdti8"
     else:
         # Sample example of how to use dse framework
         csrPluginConfig="mcycle"
@@ -152,6 +192,7 @@ def dse(csrPluginConfig, bypass, cfu, dCacheSize, hardwareDiv,
         singleCycleShift=True
         singleCycleMulDiv=True
         TARGET = "digilent_arty"  
+        WORKLOAD = "pdti8"
 
     dse(csrPluginConfig, bypass, cfu, dCacheSize, hardwareDiv, iCacheSize, 
-        mulDiv, prediction, safe, singleCycleShift, singleCycleMulDiv, TARGET)
+        mulDiv, prediction, safe, singleCycleShift, singleCycleMulDiv, TARGET, WORKLOAD)

proj/embench_template/Makefile

@@ -31,10 +31,10 @@ DEFINES += NDEBUG
 #DEFINES += INCLUDE_MODEL_MAGIC_WAND
 #DEFINES += INCLUDE_MODEL_MNV2
 #DEFINES += INCLUDE_MODEL_HPS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
 
 # Uncomment to include all TFLM examples (pdti8, micro_speech, magic_wand)
 #DEFINES += INCLUDE_ALL_TFLM_EXAMPLES

proj/example_cfu/Makefile

@@ -31,10 +31,10 @@ DEFINES += INCLUDE_MODEL_PDTI8
 #DEFINES += INCLUDE_MODEL_MAGIC_WAND
 #DEFINES += INCLUDE_MODEL_MNV2
 #DEFINES += INCLUDE_MODEL_HPS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
 
 # Uncomment to include all TFLM examples (pdti8, micro_speech, magic_wand)
 #DEFINES += INCLUDE_ALL_TFLM_EXAMPLES

proj/fccm_tutorial/Makefile

@@ -39,10 +39,10 @@ DEFINES += NDEBUG
 #DEFINES += INCLUDE_MODEL_MAGIC_WAND
 #DEFINES += INCLUDE_MODEL_MNV2
 #DEFINES += INCLUDE_MODEL_HPS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
-DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
 
 # Uncomment to include all TFLM examples (pdti8, micro_speech, magic_wand)
 #DEFINES += INCLUDE_ALL_TFLM_EXAMPLES

proj/hps_accel/Makefile

@@ -62,10 +62,10 @@ DEFINES += TF_LITE_MCU_DEBUG_LOG
 #DEFINES += INCLUDE_MODEL_MAGIC_WAND
 #DEFINES += INCLUDE_MODEL_MNV2
 DEFINES += INCLUDE_MODEL_HPS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
 
 # Uncomment to include all TFLM examples (pdti8, micro_speech, magic_wand)
 #DEFINES += INCLUDE_ALL_TFLM_EXAMPLES

proj/kws_micro_accel/Makefile

@@ -40,10 +40,10 @@ DEFINES += ALL_OPTIMIZATIONS
 #DEFINES += INCLUDE_MODEL_MAGIC_WAND
 #DEFINES += INCLUDE_MODEL_MNV2
 #DEFINES += INCLUDE_MODEL_HPS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
-DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
 
 # Uncomment to include all TFLM examples (pdti8, micro_speech, magic_wand)
 #DEFINES += INCLUDE_ALL_TFLM_EXAMPLES

proj/mport/Makefile

@@ -31,10 +31,10 @@ DEFINES += INCLUDE_MODEL_PDTI8
 #DEFINES += INCLUDE_MODEL_MAGIC_WAND
 #DEFINES += INCLUDE_MODEL_MNV2
 #DEFINES += INCLUDE_MODEL_HPS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
 
 # Uncomment to include all TFLM examples (pdti8, micro_speech, magic_wand)
 #DEFINES += INCLUDE_ALL_TFLM_EXAMPLES

proj/proj_template/Makefile

@@ -31,10 +31,10 @@ DEFINES += INCLUDE_MODEL_PDTI8
 #DEFINES += INCLUDE_MODEL_MAGIC_WAND
 #DEFINES += INCLUDE_MODEL_MNV2
 #DEFINES += INCLUDE_MODEL_HPS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
 
 # Uncomment to include all TFLM examples (pdti8, micro_speech, magic_wand)
 #DEFINES += INCLUDE_ALL_TFLM_EXAMPLES

proj/proj_template_v/Makefile

@@ -31,10 +31,10 @@ DEFINES += INCLUDE_MODEL_PDTI8
 #DEFINES += INCLUDE_MODEL_MAGIC_WAND
 #DEFINES += INCLUDE_MODEL_MNV2
 #DEFINES += INCLUDE_MODEL_HPS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_KWS
-#DEFINES += INLCUDE_MODEL_MLCOMMONS_TINY_V01_VWW
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_ANOMD
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_IMGC
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_KWS
+#DEFINES += INCLUDE_MODEL_MLCOMMONS_TINY_V01_VWW
 
 # Uncomment to include all TFLM examples (pdti8, micro_speech, magic_wand)
 #DEFINES += INCLUDE_ALL_TFLM_EXAMPLES