pkg/cmsis-nn: add support to RIOT

pkg/cmsis-nn/Makefile

@@ -0,0 +1,10 @@
+PKG_NAME=cmsis-nn
+PKG_URL=https://github.com/ARM-software/CMSIS_5
+PKG_VERSION=5.6.0
+PKG_LICENSE=Apache-2.0
+CFLAGS += -Wno-strict-aliasing -Wno-unused-parameter
+
+include $(RIOTBASE)/pkg/pkg.mk
+
+all:
+	"$(MAKE)" -C $(PKG_BUILDDIR) -f $(CURDIR)/Makefile.$(PKG_NAME)

pkg/cmsis-nn/Makefile.cmsis-nn

@@ -0,0 +1,42 @@
+PKG_NAME=cmsis-nn
+
+# A list of all the directories to build for CMSIS-NN
+CMSIS_DIRS += \
+  CMSIS/NN/Source/ActivationFunctions \
+  CMSIS/NN/Source/BasicMathFunctions \
+  CMSIS/NN/Source/ConcatenationFunctions \
+  CMSIS/NN/Source/ConvolutionFunctions \
+  CMSIS/NN/Source/FullyConnectedFunctions \
+  CMSIS/NN/Source/NNSupportFunctions \
+  CMSIS/NN/Source/PoolingFunctions \
+  CMSIS/NN/Source/ReshapeFunctions \
+  CMSIS/NN/Source/SoftmaxFunctions \
+#
+
+INCLUDES += -I$(CURDIR)/CMSIS/NN/Include
+CMSIS_BINDIRS = $(addprefix $(BINDIR)/$(PKG_NAME)/,$(CMSIS_DIRS))
+
+# Override default RIOT search path for sources to include all of the CMSIS-NN
+# sources in one library instead of one library per subdirectory.
+SRC := $(foreach DIR,$(CMSIS_DIRS),$(wildcard $(DIR)/*.c))
+SRCXX := $(foreach DIR,$(CMSIS_DIRS),$(wildcard $(DIR)/*.cpp))
+ASMSRC := $(foreach DIR,$(CMSIS_DIRS),$(wildcard $(DIR)/*.s))
+ASSMSRC := $(foreach DIR,$(CMSIS_DIRS),$(wildcard $(DIR)/*.S))
+
+OBJC    := $(SRC:%.c=$(BINDIR)/$(PKG_NAME)/%.o)
+OBJCXX  := $(SRCXX:%.cpp=$(BINDIR)/$(PKG_NAME)/%.o)
+ASMOBJ  := $(ASMSRC:%.s=$(BINDIR)/$(PKG_NAME)/%.o)
+ASSMOBJ := $(ASSMSRC:%.S=$(BINDIR)/$(PKG_NAME)/%.o)
+OBJ = $(OBJC) $(OBJCXX) $(ASMOBJ) $(ASSMOBJ)
+
+# Create subdirectories if they do not already exist
+$(OBJ): | $(CMSIS_BINDIRS)
+
+$(CMSIS_BINDIRS):
+	@mkdir -p $@
+
+# Reset the default goal.
+.DEFAULT_GOAL :=
+
+# Include RIOT settings and recipes
+include $(RIOTBASE)/Makefile.base

pkg/cmsis-nn/Makefile.dep

@@ -0,0 +1 @@
+FEATURES_REQUIRED += arch_cortexm

pkg/cmsis-nn/Makefile.include

@@ -0,0 +1,6 @@
+INCLUDES += -I$(PKGDIRBASE)/cmsis-nn/CMSIS/NN/Include
+
+# Required for some basic math functions
+INCLUDES += -I$(PKGDIRBASE)/cmsis-nn/CMSIS/DSP/Include
+
+CFLAGS += -Wno-sign-compare

tests/pkg_cmsis-nn/Makefile

@@ -0,0 +1,22 @@
+BOARD ?= nucleo-l476rg
+include ../Makefile.tests_common
+
+USEPKG += cmsis-nn
+
+BLOBS += input
+
+# Boards that were tested and are known to work
+# This package only works with Cortex M3, M4 and M7 CPUs but there's no easy
+# way provided by the build system to filter them at that level (arch_cortexm is
+# the only feature available) for the moment.
+BOARD_WHITELIST := \
+  b-l475e-iot01a \
+  iotlab-m3 \
+  nrf52832-mdk \
+  nrf52dk \
+  nucleo-l476rg \
+  same54-xpro \
+  stm32f723e-disco
+  #
+
+include $(RIOTBASE)/Makefile.include

tests/pkg_cmsis-nn/README.md

@@ -0,0 +1,31 @@
+## ARM CMSIS-NN package
+
+This application shows how to use the neural network API provided by the ARM CMSIS
+package in order to determine the type of "object" present in an RGB image.
+The image are part of the [SIFAR10 dataset](http://www.cs.toronto.edu/~kriz/cifar.html)
+which contains 10 classes of objects: plane, car, cat, bird, deer, dog, frog,
+horse, ship and truck.
+
+Expected output
+---------------
+
+```
+Predicted class: cat
+```
+
+Change the input image
+----------------------
+
+Use the `generate_image.py` script and the `-i` option to generate a new
+input image.
+For example, the following command
+```
+./generate_image.py -i 1
+```
+will generate an input containing an image with a boat.
+
+The generated image is displayed at the end of the script execution, for visual
+validation of the prediction made by the neural network running on the device.
+
+Note that each time a new image is generated, the firmware must be rebuilt so
+that it embeds the new image.

tests/pkg_cmsis-nn/generate_image.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+"""Generate a binary file from a sample image of the CIFAR-10 dataset.
+Pixel of the sample are stored as uint8, images have size 32x32x3.
+"""
+
+import os
+import argparse
+
+import numpy as np
+import matplotlib.pyplot as plt
+from tensorflow.keras.datasets import cifar10
+
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def main(args):
+    _, (cifar10_test, _) = cifar10.load_data()
+    data = cifar10_test[args.index]
+    data = data.astype('uint8')
+
+    output_path = os.path.join(SCRIPT_DIR, args.output)
+    np.ndarray.tofile(data, output_path)
+
+    if args.no_plot is False:
+        plt.imshow(data)
+        plt.show()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--index", type=int, default=0,
+                        help="Image index in CIFAR test dataset")
+    parser.add_argument("-o", "--output", type=str, default='input',
+                        help="Output filename")
+    parser.add_argument("--no-plot", default=False, action='store_true',
+                        help="Disable image display in matplotlib")
+    main(parser.parse_args())

tests/pkg_cmsis-nn/input

@@ -0,0 +1 @@
+�p1�o/�t3�v5�p.�m)�s/�q-�o,�q)�t)�o4�o1�u)�u-�w,�u(�q&�o'�p+�m,�k-�k-�j+�k,�e'�b+�a)�a&�_$~[$tU!�p3�n(�r-�t8�p1�q+�u/�r-�t.�n&�o)�n6�q4�w)�u(�s!�s�o!�p)�s2�n5�h7�f4�d0�i2�f.�f-�a&�b"�_}[ wX"�n/�m!�o$�o0�j*�s,�u-�u-�s+�s+�r0�m9�o3�s&�r%�q#�t'�r/�o6y`1nZ4bN2eM/rU2xV0�`7�g3�c'�c#�b"�_"xY!�k(�n �m�p,�n+�u.�x0�w0�s,�u-�{9��_��K�o)�s/�r6�p:�oChP/gWAbZL\ZTPKBJ?2VF4S>'qU-�b.�f+�e'�c'^$�k)�r0�s1�r/�r+�q(�x/�t,�q)�t*��;�֤Ük�r8�o<�lGoP2N5UE8qgbpnojrvafi]^]JHCTNFUI/iS-�`0�e.�^$�]$�m6�h@�d9�p5�s,�q'�t)�s)�o%�t'�v*��U�zN�f:aK+B2E:+B8-YSLvqnzyxwzzrtt^``cda[[V::/C:%lT1�i:�b,�_(d9m_P/%XJ�u0�v+�s(�v+�u*�t%�x'�k4�b;lKdWFDC9NSHHK@STJ��y���|vlicZkf^sogUSM?GE./'O=$�b:�c0�]'�sZc`\*+&F@)�o8�u*�r$�t'�w1�q3�m3x^1�nM��ktj]XWO[_XUXRMME|vk����|pf]QjbXd]TUQJ6<:1519/ kS2�g3�a'���gii6:;|yq�|R�q+�u)�z2�yB��_qY;}iN��y���yoeVPJRQMTURPNIQG=�}p��{qg]WOFSMEVRLGIC895(#J;#�j;�g-������^di�����p�t3�t/�v<ϴ����ϴ���w�����}}nk]UOVTOJJG;95LD:�}p��z�|rjbYVQJWUNTUNKLG21+(_K,�g9���ltz��������z�p2�v3zY/�ų���ܿ����������}lhxohNLEPPM-,([UM���������kd\WSMgf`XXONOI;;;)$!;.hQ.���dlt����������x;�{7�],u_P¶�ǫ�����������wjuk_fbYTTO&&"}yq���������]YRSPK^]Xhh^UWQIKN757>70L8���Z`i���������{D�{5�m/aD,�����~�r^����ʷ���{qbxrisrm22/���»����{vo[XSTSOTTP__UVWQTWYIIIOJ@I7���]_g������wnb�jB�|:�t2gH'��x�}g�������ȴ����u~ukurmGGD���������rnhWUPPPLHIFPPHcd^decZXQaYE^I"���lkp������immcYC�w>�z7dJ"sjX�{gƹ���������������}gd_GGF����������znmiUVS[][_`Zmnhstod`PaU5u_/��ń�����������NSM�xX�}Ms^4�x]��t�����摊�����ypypelh__XK��v������pleWUPGHDWXWihcpmcxn]gV6y`0�h0��̒�����������NVZ~}~�~q�yR`P%�����������uqjXqjZee\ieWpZ:��h��m��~��vmiaNLHOOM^]^e[RkS7}X-�l7�h.��ף�������°��^fi``f�����oj]=�tivi_rfYtiYf[IsnbV[Xeg_��fv`@D8 �xi�~sKE=<83:85GFAf]Nt^@�pD�t@�n6��Ͳ�����������|��VX`��������ohZPM@7�yl|o`�ud���U\]\`]��x�u]um\kcVKD:@;4,)'A><VE(�i;�w>�x6�s-�o.������������������VZcwy�z|������~F;3�vllaV��{���tvvIKI�wg�|i��vYVN31,432/14ZZ]y[<�vD�y@�q4�o2�k.������������������cgo������gio]ZWPMI]ZVzvt���������ddeYNBWM?<=9.46&.3!).9E<GSldK�}R�{L�m=qExi?ux|��ȱ�����������SW[��������ᅌ���������������Ũ��}��nmm=>>#1:"6D1FW:Qf=Un:ToEczHewNhxE`p;\p7ZsOi���ծ�����������mpq������������|��r��|��t��z��h|�D]wDWh<Re4To2Tn3Us8]}8^�3[�+`�3h�;l�0a�+a�*_�)Y�`��������������������������n��<Xo5Pi1Li1Kk0He-Os*Qx.Qq*Rt&V}.Z}.Y~+W�*Y�.]�.^�2`�7`�5^�3_�-Z�[�W�;f����������½����������=^2Tv2Tw3Uy1Sx2Tt/Vu*Tu'Rs"Oq#Sx'V}&U}*Y�-\�8g�>g�;e�8f�2c�.^�3g�0o�^�"U|Ij����������������B]v6[�2X}4Z4Z.Sy-Rs+Rq)Qp$Pq'Su(V{(Y�+\�._�;l�>n�@m�;l�6l�2i�F{�S��4r�#c�V�)SzB_~���������|��:\r1W{8^�6\�,Rw,Rw/Sw.Tw+Sw+V{,X,Z�-a�6j�:n�6i�.a�+_�$[�3l�I��U��L}�2n�#b�Y�#V�,S~Nj�������a~�Ah~6^�0W|:a�0W{(Pt-Rw/Tz0W~/Y�.Y�3a�'\�']�0f�/e�']�U�(e�C��C~�.b�3`�2l�#a� \�!X�)X�.T}h�����@dw6ay4^�5_�=g�:d�6`�-Sx*Ov)Px.X�1\�.\�*_�(]�'\�%Z�(]�,f�?}�/n�Z�<g3]�D|�*d�X�&[�%W�+Y�*OqGk�1YrMiGi&Ru1]�8d�:f�5\�8^�<c�9c�5a�2_�-^�'X�!S}*[�>p�O��I��8t�&a�@l(U=t�1f�#U�+[�'Z�*\�,X}(Qp*UsHhCfJmGjIl$Ps/Vx8_�>e�Bm�Kw�Eq�1_�+X+X<i�U��m��]��<s�R�R~@k6k�8i�-Y�+V�(Y�(\�(W{&Qs$OrEiBeIlEhIl?b:Y Fd/Wv=h�Jw�Bo�5`�4_�-W{Cm�Y��i��Y��0c�M|"T�Cn

tests/pkg_cmsis-nn/main.c

@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2019 Inria
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup         tests
+ * @{
+ *
+ * @file
+ * @brief           Sample application for ARM CMSIS-NN package
+ *
+ * This example is adapted from ARM CMSIS CIFAR10 example to RIOT by Alexandre Abadie
+ * https://github.com/ARM-software/CMSIS_5/tree/develop/CMSIS/NN/Examples/ARM/arm_nn_examples/cifar10
+ *
+ * @author          Alexandre Abadie <[email protected]>
+ */
+
+#include <stdint.h>
+#include <stdio.h>
+#include "arm_math.h"
+#include "parameter.h"
+#include "weights.h"
+
+#include "arm_nnfunctions.h"
+
+#include "blob/input.h"
+
+/* There are 10 different classes of objects in the CIFAR10 dataset */
+#define CLASSES_NUMOF   10
+
+/* include the input and weights */
+static const q7_t conv1_wt[CONV1_IM_CH * CONV1_KER_DIM * CONV1_KER_DIM * CONV1_OUT_CH] = CONV1_WT;
+static const q7_t conv1_bias[CONV1_OUT_CH] = CONV1_BIAS;
+
+static const q7_t conv2_wt[CONV2_IM_CH * CONV2_KER_DIM * CONV2_KER_DIM * CONV2_OUT_CH] = CONV2_WT;
+static const q7_t conv2_bias[CONV2_OUT_CH] = CONV2_BIAS;
+
+static const q7_t conv3_wt[CONV3_IM_CH * CONV3_KER_DIM * CONV3_KER_DIM * CONV3_OUT_CH] = CONV3_WT;
+static const q7_t conv3_bias[CONV3_OUT_CH] = CONV3_BIAS;
+
+static const q7_t ip1_wt[IP1_DIM * IP1_OUT] = IP1_WT;
+static const q7_t ip1_bias[IP1_OUT] = IP1_BIAS;
+
+/* Here the image_data should be the raw uint8 type RGB image in [RGB, RGB, RGB ... RGB] format */
+// static const uint8_t image_data[CONV1_IM_CH * CONV1_IM_DIM * CONV1_IM_DIM] = IMG_DATA;
+static q7_t output_data[IP1_OUT];
+
+/* vector buffer: max(im2col buffer,average pool buffer, fully connected buffer) */
+static q7_t col_buffer[2 * 5 * 5 * 32 * 2];
+static q7_t img_buffer1[32 * 32 * 10 * 4];
+static q7_t *img_buffer2 = (q7_t *)(img_buffer1 + (32 * 32 * 32));
+
+static const char classes[CLASSES_NUMOF][6] = {
+    "plane", "car", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck" };
+
+int main(void)
+{
+    printf("start execution\n");
+
+    uint8_t *image_data = (uint8_t *)input;
+
+    /* input pre-processing */
+    int mean_data[3] = INPUT_MEAN_SHIFT;
+    unsigned int scale_data[3] = INPUT_RIGHT_SHIFT;
+    for (unsigned i = 0; i < input_len; i += 3) {
+        img_buffer2[i]     = (q7_t)__SSAT( ((((int)image_data[i]     - mean_data[0]) << 7) + (0x1 << (scale_data[0] - 1)))
+                                 >> scale_data[0], 8);
+        img_buffer2[i + 1] = (q7_t)__SSAT( ((((int)image_data[i + 1] - mean_data[1]) << 7) + (0x1 << (scale_data[1] - 1)))
+                                 >> scale_data[1], 8);
+        img_buffer2[i + 2] = (q7_t)__SSAT( ((((int)image_data[i + 2] - mean_data[2]) << 7) + (0x1 << (scale_data[2] - 1)))
+                                 >> scale_data[2], 8);
+    }
+
+    /* conv1 img_buffer2 -> img_buffer1 */
+    arm_convolve_HWC_q7_RGB(img_buffer2, CONV1_IM_DIM, CONV1_IM_CH, conv1_wt, CONV1_OUT_CH, CONV1_KER_DIM, CONV1_PADDING,
+                            CONV1_STRIDE, conv1_bias, CONV1_BIAS_LSHIFT, CONV1_OUT_RSHIFT, img_buffer1, CONV1_OUT_DIM,
+                            (q15_t *)col_buffer, NULL);
+
+    arm_relu_q7(img_buffer1, CONV1_OUT_DIM * CONV1_OUT_DIM * CONV1_OUT_CH);
+
+    /* pool1 img_buffer1 -> img_buffer2 */
+    arm_maxpool_q7_HWC(img_buffer1, CONV1_OUT_DIM, CONV1_OUT_CH, POOL1_KER_DIM,
+                       POOL1_PADDING, POOL1_STRIDE, POOL1_OUT_DIM, NULL, img_buffer2);
+
+    /* conv2 img_buffer2 -> img_buffer1 */
+    arm_convolve_HWC_q7_fast(img_buffer2, CONV2_IM_DIM, CONV2_IM_CH, conv2_wt, CONV2_OUT_CH, CONV2_KER_DIM,
+                             CONV2_PADDING, CONV2_STRIDE, conv2_bias, CONV2_BIAS_LSHIFT, CONV2_OUT_RSHIFT, img_buffer1,
+                             CONV2_OUT_DIM, (q15_t *)col_buffer, NULL);
+
+    arm_relu_q7(img_buffer1, CONV2_OUT_DIM * CONV2_OUT_DIM * CONV2_OUT_CH);
+
+    /* pool2 img_buffer1 -> img_buffer2 */
+    arm_maxpool_q7_HWC(img_buffer1, CONV2_OUT_DIM, CONV2_OUT_CH, POOL2_KER_DIM,
+                       POOL2_PADDING, POOL2_STRIDE, POOL2_OUT_DIM, col_buffer, img_buffer2);
+
+  /* conv3 img_buffer2 -> img_buffer1 */
+    arm_convolve_HWC_q7_fast(img_buffer2, CONV3_IM_DIM, CONV3_IM_CH, conv3_wt, CONV3_OUT_CH, CONV3_KER_DIM,
+                             CONV3_PADDING, CONV3_STRIDE, conv3_bias, CONV3_BIAS_LSHIFT, CONV3_OUT_RSHIFT, img_buffer1,
+                             CONV3_OUT_DIM, (q15_t *)col_buffer, NULL);
+
+    arm_relu_q7(img_buffer1, CONV3_OUT_DIM * CONV3_OUT_DIM * CONV3_OUT_CH);
+
+    /* pool3 img_buffer-> img_buffer2 */
+    arm_maxpool_q7_HWC(img_buffer1, CONV3_OUT_DIM, CONV3_OUT_CH, POOL3_KER_DIM,
+                       POOL3_PADDING, POOL3_STRIDE, POOL3_OUT_DIM, col_buffer, img_buffer2);
+
+    arm_fully_connected_q7_opt(img_buffer2, ip1_wt, IP1_DIM, IP1_OUT, IP1_BIAS_LSHIFT, IP1_OUT_RSHIFT, ip1_bias,
+                               output_data, (q15_t *)img_buffer1);
+
+    arm_softmax_q7(output_data, CLASSES_NUMOF, output_data);
+
+    int val = -1;
+    uint8_t class_idx = 0;
+    for (unsigned i = 0; i < CLASSES_NUMOF; i++) {
+        if (output_data[i] > val) {
+            val = output_data[i];
+            class_idx = i;
+        }
+    }
+
+    if (val > 0) {
+        printf("Predicted class: %s\n", classes[class_idx]);
+    }
+    else {
+        puts("No match found");
+    }
+
+    return 0;
+}

tests/pkg_cmsis-nn/parameter.h

@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2019 Inria
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup         tests
+ * @{
+ *
+ * @file
+ * @brief           CNN parameters
+ */
+
+#ifndef PARAMETER_H
+#define PARAMETER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CONV1_IM_DIM 32
+#define CONV1_IM_CH 3
+#define CONV1_KER_DIM 5
+#define CONV1_PADDING 2
+#define CONV1_STRIDE 1
+#define CONV1_OUT_CH 32
+#define CONV1_OUT_DIM 32
+
+#define POOL1_KER_DIM 3
+#define POOL1_STRIDE 2
+#define POOL1_PADDING 0
+#define POOL1_OUT_DIM 16
+
+#define CONV2_IM_DIM 16
+#define CONV2_IM_CH 32
+#define CONV2_KER_DIM 5
+#define CONV2_PADDING 2
+#define CONV2_STRIDE 1
+#define CONV2_OUT_CH 16
+#define CONV2_OUT_DIM 16
+
+#define POOL2_KER_DIM 3
+#define POOL2_STRIDE 2
+#define POOL2_PADDING 0
+#define POOL2_OUT_DIM 8
+
+#define CONV3_IM_DIM 8
+#define CONV3_IM_CH 16
+#define CONV3_KER_DIM 5
+#define CONV3_PADDING 2
+#define CONV3_STRIDE 1
+#define CONV3_OUT_CH 32
+#define CONV3_OUT_DIM 8
+
+#define POOL3_KER_DIM 3
+#define POOL3_STRIDE 2
+#define POOL3_PADDING 0
+#define POOL3_OUT_DIM 4
+
+#define IP1_DIM 4*4*32
+#define IP1_IM_DIM 4
+#define IP1_IM_CH 32
+#define IP1_OUT 10
+
+#ifdef __cplusplus
+} /* end extern "C" */
+#endif
+
+#endif /* PARAMETER_H */
+/** @} */

tests/pkg_cmsis-nn/tests/01-run.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+import sys
+from testrunner import run
+
+
+def testfunc(child):
+    child.expect_exact("Predicted class: cat")
+
+
+if __name__ == "__main__":
+    sys.exit(run(testfunc))