From 25b331351186754f68df01a4a9c2668267a010b0 Mon Sep 17 00:00:00 2001 From: Joseph Liu Date: Tue, 30 May 2017 19:01:06 +0800 Subject: [PATCH 01/41] Initial LINE notification support --- camera.js | 2 + config.js | 6 +++ line.js | 99 +++++++++++++++++++++++++++++++++++++++++ package.json | 2 + utils/local-launcher.js | 6 +++ 5 files changed, 115 insertions(+) create mode 100644 line.js diff --git a/camera.js b/camera.js index 2b9c519..55fc01a 100644 --- a/camera.js +++ b/camera.js @@ -28,6 +28,7 @@ const client = mqtt.connect(broker); const topicActionLog = config.topicActionLog; const topicActionInference = config.topicActionInference; const topicEventCamera = config.topicEventCamera; +const topicNotifyLINE = config.topicNotifyLINE; const cameraURI = config.ipcameraSnapshot; const snapshotFile = '/tmp/snapshot.jpg'; const cameraCmd = '/usr/bin/raspistill'; @@ -58,6 +59,7 @@ client.on('message', (t, m) => { log('camera client: cannot get image.'); } else { log('camera client: publishing image.'); + client.publish(topicNotifyLINE, data); client.publish(topicActionInference, data); } }); diff --git a/config.js b/config.js index b4bf420..3d6b6eb 100644 --- a/config.js +++ b/config.js @@ -41,6 +41,7 @@ config.topicEventCamera = padTopicBase('event/camera'); config.topicEventLocalImage = padTopicBase('event/localImage'); config.topicNotifyEmail = padTopicBase('notify/email'); config.topicNotifySMS = padTopicBase('notify/sms'); +config.topicNotifyLINE = padTopicBase('notify/line'); config.topicDashboardLog = padTopicBase('dashboard/log'); config.topicDashboardSnapshot = padTopicBase('dashboard/snapshot'); config.topicDashboardInferenceResult = padTopicBase('dashboard/inferenceResult'); @@ -58,5 +59,10 @@ config.sender_email = config.senderEmail; config.sender_password = config.senderPassword; config.receiver_email = config.receiverEmail; +// Authentication and channel information for LINE +config.LINEChannelID = 'LINE_CHANNEL_ID'; +config.LINEChannelSecret = 'LINE_CHANNEL_SECRET'; +config.LINEChannelAccessToken = 'LINE_CHANNEL_ACCESS_TOKEN'; + // make config importable module.exports = config; diff --git a/line.js b/line.js new file mode 100644 index 0000000..24b7dc6 --- /dev/null +++ b/line.js @@ -0,0 +1,99 @@ +// Copyright 2017 DT42 +// +// This file is part of BerryNet. +// +// BerryNet is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// BerryNet is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with BerryNet. If not, see . + +// Usage: +// ./mail.js +// This app assumes the user uses gmail. +// You may need to configure "Allow Less Secure Apps" in your Gmail account. + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const moment = require('moment'); +const mqtt = require('mqtt'); +const line = require('@line/bot-sdk'); +const imgur = require('imgur'); +const config = require('./config'); + +const broker = config.brokerHost; +const client = mqtt.connect(broker); +const topicActionLog = config.topicActionLog; +const topicNotifyLINE = config.topicNotifyLINE; +const channelID = config.LINEChannelID; + +// create LINE SDK config +const LINEConfig = { + channelAccessToken: config.LINEChannelAccessToken, + channelSecret: config.LINEChannelSecret, +}; + +// create LINE SDK client +const LINEClient = new line.Client(LINEConfig); + +function log(m) { + client.publish(topicActionLog, m); + console.log(m); +} + +function saveBufferToImage(b, filepath) { + fs.writeFile(filepath, b, (e) => { + if (e) + log(`LINE client: cannot save buffer to image.`); + else + log(`LINE client: saved buffer to image ${filepath} successfully.`); + }); +} + +client.on('connect', () => { + client.subscribe(topicNotifyLINE); + log(`LINE client: connected to ${broker} successfully.`); +}); + +client.on('message', (t, m) => { + const size = m.length; + log(`LINE client: on topic ${t}, received ${size} bytes.`) + + // save image to file and upload it to imgur for display in LINE message + const now = moment().format('YYYYMMDD-HHmmss'); + const snapshot = `snapshot-${now}.jpg` + const snapshot_path = path.join('/tmp', snapshot) + saveBufferToImage(m, snapshot_path); + imgur.uploadFile(snapshot_path) + .then((json) => { + var imgur_link = json.data.link; + imgur_link = imgur_link.replace('http:\/\/', 'https:\/\/'); + log(`LINE client: An image has been uploaded to imgur. link: ${imgur_link}`); + + // Image can only be delivered via 'https://' URL, 'http://' doesn't work + LINEClient.pushMessage(channelID, [{ type: 'text', + text: now }, + { type: 'image', + originalContentUrl: imgur_link, + previewImageUrl: imgur_link } + ]) + .then((v) => { + log(`LINE client: message sent to ${channelID} successfully.`); + }) + .catch((err) => { + log(`LINE client: an error occurred, ${err}.`); + }); + }) + .catch((err) => { + log(`LINE client: an error occurred. ${err}`); + }); +}); diff --git a/package.json b/package.json index 5179913..79430e5 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,8 @@ "dependencies": { "ascoltatori": "^3.1.0", "emailjs": "^1.0.8", + "@line/bot-sdk": "^1.0.0", + "imgur": "^0.2.1", "imagemagick": "^0.1.3", "mocha": "^3.2.0", "mosca": "^2.2.0", diff --git a/utils/local-launcher.js b/utils/local-launcher.js index 8460bcf..36140a6 100755 --- a/utils/local-launcher.js +++ b/utils/local-launcher.js @@ -20,6 +20,7 @@ const broker = exec('node broker.js', execCallback); const cameraAgent = exec('node camera.js', execCallback); //const eventNotifier = exec('node mail.js ' + config.sender_email + ' ' + config.sender_email_password + ' ' + config.receiver_email, execCallback); const eventLogger = exec('node journal.js', execCallback); +const LINEAgent = exec('node line.js', execCallback); const webServer = exec('cd dashboard && node server.js', execCallback); //const dlClassifier = exec('cd inference && python classify_server.py --model_dir=model --image_dir=image', execCallback); const dlDetector = exec('cd inference/darkflow && python detection_server.py', execCallback); @@ -43,6 +44,10 @@ eventLogger.stdout.on('data', function(data) { console.log('[eventLogger] ' + data); }); +LINEAgent.stdout.on('data', function(data) { + console.log('[LINEAgent] ' + data); +}); + webServer.stdout.on('data', function(data) { console.log('[webServer] ' + data); }); @@ -60,6 +65,7 @@ process.on('SIGINT', function() { broker.kill(); cameraAgent.kill(); eventNotifier.kill(); + LINEAgent.kill(); eventLogger.kill(); webServer.kill(); //dlClassifier.kill(); From 0b19ac3beaebccf2d1727324cb25f6c430d88963 Mon Sep 17 00:00:00 2001 From: Joseph Liu Date: Tue, 30 May 2017 23:04:24 +0800 Subject: [PATCH 02/41] targetUserID is mis-understood as channelID. Rename variables accordingly --- config.js | 2 +- line.js | 21 ++++++++------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/config.js b/config.js index 3d6b6eb..d94d35e 100644 --- a/config.js +++ b/config.js @@ -60,7 +60,7 @@ config.sender_password = config.senderPassword; config.receiver_email = config.receiverEmail; // Authentication and channel information for LINE -config.LINEChannelID = 'LINE_CHANNEL_ID'; +config.LINETargetUserID = 'LINE_TARGET_USER_ID'; config.LINEChannelSecret = 'LINE_CHANNEL_SECRET'; config.LINEChannelAccessToken = 'LINE_CHANNEL_ACCESS_TOKEN'; diff --git a/line.js b/line.js index 24b7dc6..28f41a0 100644 --- a/line.js +++ b/line.js @@ -15,11 +15,6 @@ // You should have received a copy of the GNU General Public License // along with BerryNet. If not, see . -// Usage: -// ./mail.js -// This app assumes the user uses gmail. -// You may need to configure "Allow Less Secure Apps" in your Gmail account. - 'use strict'; const fs = require('fs'); @@ -34,7 +29,7 @@ const broker = config.brokerHost; const client = mqtt.connect(broker); const topicActionLog = config.topicActionLog; const topicNotifyLINE = config.topicNotifyLINE; -const channelID = config.LINEChannelID; +const targetUserID = config.LINETargetUserID; // create LINE SDK config const LINEConfig = { @@ -80,14 +75,14 @@ client.on('message', (t, m) => { log(`LINE client: An image has been uploaded to imgur. link: ${imgur_link}`); // Image can only be delivered via 'https://' URL, 'http://' doesn't work - LINEClient.pushMessage(channelID, [{ type: 'text', - text: now }, - { type: 'image', - originalContentUrl: imgur_link, - previewImageUrl: imgur_link } - ]) + LINEClient.pushMessage(targetUserID, [{ type: 'text', + text: now }, + { type: 'image', + originalContentUrl: imgur_link, + previewImageUrl: imgur_link } + ]) .then((v) => { - log(`LINE client: message sent to ${channelID} successfully.`); + log(`LINE client: message sent to ${targetUserID} successfully.`); }) .catch((err) => { log(`LINE client: an error occurred, ${err}.`); From 087eb8e2a566181a23867eabe54a83dce9a2dc65 Mon Sep 17 00:00:00 2001 From: Joseph Liu Date: Wed, 31 May 2017 06:16:25 +0800 Subject: [PATCH 03/41] Use the result image from object detection instead of camera for LINE notification --- camera.js | 2 +- inference/agent.js | 2 ++ journal.js | 2 +- line.js | 50 +++++++++++++++++++--------------------------- 4 files changed, 24 insertions(+), 32 deletions(-) diff --git a/camera.js b/camera.js index 55fc01a..810c87f 100644 --- a/camera.js +++ b/camera.js @@ -59,7 +59,7 @@ client.on('message', (t, m) => { log('camera client: cannot get image.'); } else { log('camera client: publishing image.'); - client.publish(topicNotifyLINE, data); + // client.publish(topicNotifyLINE, data); client.publish(topicActionInference, data); } }); diff --git a/inference/agent.js b/inference/agent.js index 4ee9879..127ebc3 100644 --- a/inference/agent.js +++ b/inference/agent.js @@ -30,6 +30,7 @@ const topicActionLog = config.topicActionLog; const topicActionInference = config.topicActionInference; const topicDashboardSnapshot = config.topicDashboardSnapshot; const topicDashboardInferenceResult = config.topicDashboardInferenceResult; +const topicNotifyLINE = config.topicNotifyLINE; const inferenceEngine = config.inferenceEngine; function log(m) { @@ -107,6 +108,7 @@ client.on('message', (t, m) => { console.log('Unknown owner ' + inferenceEngine); } + client.publish(topicNotifyLINE, dashboard_image_path); client.publish(topicDashboardInferenceResult, result.toString().replace(/(\n)+/g, '
')) }) } else { diff --git a/journal.js b/journal.js index 4d1882c..3546b90 100644 --- a/journal.js +++ b/journal.js @@ -54,7 +54,7 @@ client.on('connect', () => { client.on('message', (t, m) => { // secretly save a copy of the image - if (t == topicNotifyEmail) { + if (t === topicNotifyEmail) { const filename = 'snapshot.jpg'; saveBufferToImage(m, snapshot); client.publish(topicDashboardSnapshot, filename); diff --git a/line.js b/line.js index 28f41a0..e4b4c5e 100644 --- a/line.js +++ b/line.js @@ -17,9 +17,6 @@ 'use strict'; -const fs = require('fs'); -const path = require('path'); -const moment = require('moment'); const mqtt = require('mqtt'); const line = require('@line/bot-sdk'); const imgur = require('imgur'); @@ -29,6 +26,7 @@ const broker = config.brokerHost; const client = mqtt.connect(broker); const topicActionLog = config.topicActionLog; const topicNotifyLINE = config.topicNotifyLINE; +const topicDashboardInferenceResult = config.topicDashboardInferenceResult; const targetUserID = config.LINETargetUserID; // create LINE SDK config @@ -45,50 +43,42 @@ function log(m) { console.log(m); } -function saveBufferToImage(b, filepath) { - fs.writeFile(filepath, b, (e) => { - if (e) - log(`LINE client: cannot save buffer to image.`); - else - log(`LINE client: saved buffer to image ${filepath} successfully.`); - }); -} - client.on('connect', () => { client.subscribe(topicNotifyLINE); - log(`LINE client: connected to ${broker} successfully.`); + client.subscribe(topicDashboardInferenceResult); + log(`client connected to ${broker} successfully.`); }); client.on('message', (t, m) => { const size = m.length; - log(`LINE client: on topic ${t}, received ${size} bytes.`) + log(`client on topic ${t}, received ${size} bytes.`) + + if (t === topicDashboardInferenceResult) { + const result = m.toString(); + LINEClient.pushMessage(targetUserID, { type: 'text', text: result }); + return; + } // save image to file and upload it to imgur for display in LINE message - const now = moment().format('YYYYMMDD-HHmmss'); - const snapshot = `snapshot-${now}.jpg` - const snapshot_path = path.join('/tmp', snapshot) - saveBufferToImage(m, snapshot_path); + const snapshot_path = m.toString(); imgur.uploadFile(snapshot_path) .then((json) => { - var imgur_link = json.data.link; - imgur_link = imgur_link.replace('http:\/\/', 'https:\/\/'); - log(`LINE client: An image has been uploaded to imgur. link: ${imgur_link}`); + var imgurLink = json.data.link; + imgurLink = imgurLink.replace('http:\/\/', 'https:\/\/'); + log(`An image has been uploaded to imgur. link: ${imgurLink}`); // Image can only be delivered via 'https://' URL, 'http://' doesn't work - LINEClient.pushMessage(targetUserID, [{ type: 'text', - text: now }, - { type: 'image', - originalContentUrl: imgur_link, - previewImageUrl: imgur_link } - ]) + LINEClient.pushMessage(targetUserID, { type: 'image', + originalContentUrl: imgurLink, + previewImageUrl: imgurLink }) .then((v) => { - log(`LINE client: message sent to ${targetUserID} successfully.`); + log(`A message sent to ${targetUserID} successfully.`); }) .catch((err) => { - log(`LINE client: an error occurred, ${err}.`); + log(`An error occurred, ${err}.`); }); }) .catch((err) => { - log(`LINE client: an error occurred. ${err}`); + log(`An error occurred. ${err}`); }); }); From 2daf49da87e0c977a0ab633208bd5306ab101219 Mon Sep 17 00:00:00 2001 From: Joseph Liu Date: Wed, 31 May 2017 06:20:14 +0800 Subject: [PATCH 04/41] Add systemd support for LINE MQTT client. --- berrynet-manager | 3 ++- configure | 2 +- systemd/line.service | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 systemd/line.service diff --git a/berrynet-manager b/berrynet-manager index 39fce93..b948950 100755 --- a/berrynet-manager +++ b/berrynet-manager @@ -33,7 +33,7 @@ fi case $1 in start | stop | status) - sudo systemctl $1 detection_server.service agent.service broker.service dashboard.service localimg.service camera.service journal.service cleaner.timer + sudo systemctl $1 detection_server.service agent.service broker.service dashboard.service localimg.service camera.service journal.service line.service cleaner.timer ;; log) sudo journalctl -x --no-pager -u detection_server.service @@ -43,6 +43,7 @@ case $1 in sudo journalctl -x --no-pager -u localimg.service sudo journalctl -x --no-pager -u camera.service sudo journalctl -x --no-pager -u journal.service + sudo journalctl -x --no-pager -u line.service sudo journalctl -x --no-pager -u cleaner.timer ;; *) diff --git a/configure b/configure index a28ac0e..9146fa5 100755 --- a/configure +++ b/configure @@ -127,7 +127,7 @@ install_systemd_configs() { install_gateway() { local working_dir="/usr/local/berrynet" sudo mkdir -p $working_dir - sudo cp -a broker.js camera.js cleaner.sh config.js dashboard inference journal.js localimg.js mail.js package.json $working_dir + sudo cp -a broker.js camera.js cleaner.sh config.js line.js dashboard inference journal.js localimg.js mail.js package.json $working_dir sudo cp berrynet-manager /usr/local/bin # install npm dependencies pushd $working_dir > /dev/null diff --git a/systemd/line.service b/systemd/line.service new file mode 100644 index 0000000..f04a623 --- /dev/null +++ b/systemd/line.service @@ -0,0 +1,15 @@ +[Unit] +Description=LINE client agent for notification +After=network.target + +[Service] +Type=simple +WorkingDirectory=/usr/local/berrynet +PIDFile=/tmp/line.pid +ExecStart=/usr/bin/node line.js +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +WantedBy=graphical.target From 289ff6ac069e2cb4f5c28f51befa73133d48e48c Mon Sep 17 00:00:00 2001 From: "Ying-Chun Liu (PaulLiu)" Date: Thu, 22 Jun 2017 11:43:34 +0800 Subject: [PATCH 05/41] Add USB camera support. Signed-off-by: Ying-Chun Liu (PaulLiu) --- camera.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/camera.js b/camera.js index 2b9c519..d222332 100644 --- a/camera.js +++ b/camera.js @@ -34,6 +34,8 @@ const cameraCmd = '/usr/bin/raspistill'; const cameraArgs = ['-vf', '-hf', '-w', '1024', '-h', '768', '-o', snapshotFile]; +const usbCameraCmd = '/usr/bin/fswebcam'; +const usbCameraArgs = ['-r', '1024x768', '--no-banner', '-D', '0.5', snapshotFile]; function log(m) { client.publish(topicActionLog, m); @@ -75,6 +77,17 @@ client.on('message', (t, m) => { } } ); + } else if (action == 'snapshot_usb') { + // Take a snapshot from USB camera. + spawnsync(usbCameraCmd, usbCameraArgs); + fs.readFile(snapshotFile, function(err, data) { + if (err) { + log('camera client: cannot get image.'); + } else { + log('camera client: publishing image.'); + client.publish(topicActionInference, data); + } + }); } else { log('camera client: unkown action.'); } From 090eb90a792d961c2b61e406349112743bf4159f Mon Sep 17 00:00:00 2001 From: "Ying-Chun Liu (PaulLiu)" Date: Thu, 20 Jul 2017 01:08:45 +0800 Subject: [PATCH 06/41] Add USB camera dependencies (fswebcam) to ./configure Signed-off-by: Ying-Chun Liu (PaulLiu) --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index a28ac0e..e575acf 100755 --- a/configure +++ b/configure @@ -23,7 +23,7 @@ LOG="/tmp/berrynet.log" install_system_dependencies() { sudo apt-get update - sudo apt-get install -y python-dev python-pip python-opencv mongodb libkrb5-dev libzmq3-dev libyaml-dev imagemagick curl + sudo apt-get install -y python-dev python-pip python-opencv mongodb libkrb5-dev libzmq3-dev libyaml-dev imagemagick curl fswebcam sudo service mongodb start sudo -H pip install watchdog cython } From d247a4d305b5551e624525429209026d818df794 Mon Sep 17 00:00:00 2001 From: "Ying-Chun Liu (PaulLiu)" Date: Thu, 20 Jul 2017 01:10:43 +0800 Subject: [PATCH 07/41] Update README.md for capturing picture from USB webcam. Signed-off-by: Ying-Chun Liu (PaulLiu) --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c8f726f..863029f 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,12 @@ To capture an image via configured IP camera $ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_ipcam ``` +To capture an image via USB webcam + +``` +$ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_usb +``` + To provide a local image ``` From 3c3aff2569ae82c6aa44ebd2ee96ad30e7d79a73 Mon Sep 17 00:00:00 2001 From: Tammy Date: Sat, 22 Jul 2017 18:25:16 +0800 Subject: [PATCH 08/41] update README and add xmlTotxt.py to utils --- README.md | 15 +++++ utils/xmlTotxt.py | 153 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 utils/xmlTotxt.py diff --git a/README.md b/README.md index c8f726f..cec0c21 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,21 @@ To provide a local image $ mosquitto_pub -h localhost -t berrynet/event/localImage -m ``` +# Use Your Data To Train + +The original instruction of retraining YOLOv2 model see [github repository of darknet](https://github.com/AlexeyAB/darknet#how-to-train-to-detect-your-custom-objects) + +In the current of BerryNet, TinyYolo is used instead of YOLOv2. +The major differences are: + +1. Create file yolo-obj.cfg with the same content as in `tiny-yolo.cfg` +2. Download pre-trained weights of darknet reference model, `darknet.weights.12`, for the convolutional layers (6.1MB) +https://drive.google.com/drive/folders/0B-oZJEwmkAObMzAtc2QzZDhyVGM?usp=sharing + +The rest parts are the same as retraining YOLO. + +If you use [LabelMe](http://labelme.csail.mit.edu/Release3.0/) to annotate data, `utils/xmlTotxt.py` can help convert the xml format to the text format that darknet uses. + # Discussion diff --git a/utils/xmlTotxt.py b/utils/xmlTotxt.py new file mode 100644 index 0000000..4ae419c --- /dev/null +++ b/utils/xmlTotxt.py @@ -0,0 +1,153 @@ +''' + +$python xmlTotxt.py -n $FOLDER -u $LABELME_USER + --classes $CLASS_1 $CLASS_2 $CLASS_3... + +''' +import os +from os.path import join +import argparse +import logging +from shutil import copyfile +import xml.etree.ElementTree as ET + + +def convert(size, X, Y): + dw = 1./size[0] + dh = 1./size[1] + x = (X[0] + X[1])/2.0 + y = (Y[0] + Y[1])/2.0 + w = X[1] - X[0] + h = Y[1] - Y[0] + x = x*dw + w = w*dw + y = y*dh + h = h*dh + return (x, y, w, h) + + +def convert_annotation(in_dir, out_dir, image_id, out_id, classes): + in_file = open('%s/%s.xml' % (in_dir, image_id)) + o_file = open(out_dir + '/%s.txt' % out_id, 'w') + tree = ET.parse(in_file) + root = tree.getroot() + size = root.find('imagesize') + h = float(size.find('nrows').text) + w = float(size.find('ncols').text) + + for obj in root.iter('object'): + X = [] + Y = [] + cls = obj.find('name').text + if cls not in classes: + logging.debug("%s is not in the selected class" % cls) + continue + cls_id = classes.index(cls) + for pt in obj.find('polygon').findall('pt'): + X.append(float(pt.find('x').text)) + Y.append(float(pt.find('y').text)) + if (len(X) < 2 or len(Y) < 2): + logging.warning("%s doesn't have sufficient info, ignore" % cls) + continue + X = list(set(X)) + X.sort() + Y = list(set(Y)) + Y.sort() + bb = convert((w, h), X, Y) + o_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n') + + +def find_output_id(out_img_dir, image_id, suffix): + counter = 0 + out_img_path = join(out_img_dir, image_id + "." + suffix) + out_id = image_id + while os.path.exists(out_img_path): + logging.info("%s exists" % out_img_path) + counter += 1 + out_id = image_id + "_" + str(counter) + out_img_path = join(out_img_dir, out_id + "." + suffix) + return out_id, out_img_path + + +def main(): + + parser = argparse.ArgumentParser( + description="Simple tool to make the scene image better." + ) + parser.add_argument( + "-v", "--verbosity", action="count", + help="increase output verbosity" + ) + parser.add_argument( + "-r", "--root", type=str, default=None, + help="Specify the root directory (default: PWD)" + ) + parser.add_argument( + "-n", "--name", type=str, default="youtube_09", + help="Specify the name of the original video (default: youtube_09)" + ) + parser.add_argument( + "-u", "--user", type=str, default="V", + help="Specify the username (default: V)" + ) + parser.add_argument( + "-o", "--outdir", type=str, default="test2017", + help="Output dir in root, must end with 2017 (default: test2017)" + ) + parser.add_argument( + "--classes", nargs="+", type=str, default=["fighting", "dog"], + help="Classes to be trained. Default: [fighting, dog]" + ) + parser.add_argument( + "-d", "--delete", action="store_true", + help="Use this option to clean up train.txt" + ) + + args = parser.parse_args() + + log_level = logging.WARNING + if args.verbosity == 1: + log_level = logging.INFO + elif args.verbosity >= 2: + log_level = logging.DEBUG + logging.basicConfig(level=log_level, + format='[xmlTotxt: %(levelname)s] %(message)s') + + logging.info(args.classes) + + if args.root is None: + args.root = os.getcwd() + in_dir = join(args.root, "Annotations", "users", + args.user, args.name) + in_img_dir = join(args.root, "Images", "users", + args.user, args.name) + out_dir = join(args.root, args.outdir, "labels") + out_img_dir = join(args.root, args.outdir, "JPEGImages") + + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + if not os.path.exists(out_img_dir): + os.makedirs(out_img_dir) + + foutput = join(args.root, "train.txt") + if args.delete: + output = open(foutput, 'w') + else: + output = open(foutput, 'a') + + for _img_path in os.listdir(in_img_dir): + img_path = os.path.join(in_img_dir, _img_path) + suffix = os.path.basename(_img_path).split('.')[1] + logging.debug("Find %s" % img_path) + image_id = os.path.basename(_img_path).split('.')[0] + out_id, out_img_path = find_output_id(out_img_dir, image_id, suffix) + convert_annotation(in_dir, out_dir, image_id, out_id, args.classes) + logging.info("Copy %s to %s" % (img_path, out_img_path)) + copyfile(img_path, out_img_path) + + output.write(out_img_path + '\n') + + +if __name__ == '__main__': + main() From d60c1396e2c3f93849acc0fad6b01d2d0801c0be Mon Sep 17 00:00:00 2001 From: Tammy Date: Mon, 24 Jul 2017 17:13:32 +0800 Subject: [PATCH 09/41] based on the results of code review, fix format issues --- utils/xmlTotxt.py | 52 +++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/utils/xmlTotxt.py b/utils/xmlTotxt.py index 4ae419c..efa0690 100644 --- a/utils/xmlTotxt.py +++ b/utils/xmlTotxt.py @@ -1,9 +1,13 @@ -''' +""" $python xmlTotxt.py -n $FOLDER -u $LABELME_USER --classes $CLASS_1 $CLASS_2 $CLASS_3... -''' +FOLDER: LabelMe project folder name +LABELME_USER: LabelMe user name +CLASS_i: Class labels defined on LabelMe + +""" import os from os.path import join import argparse @@ -12,13 +16,13 @@ import xml.etree.ElementTree as ET -def convert(size, X, Y): +def convert(size, in_x, in_y): dw = 1./size[0] dh = 1./size[1] - x = (X[0] + X[1])/2.0 - y = (Y[0] + Y[1])/2.0 - w = X[1] - X[0] - h = Y[1] - Y[0] + x = (in_x[0] + in_x[1])/2.0 + y = (in_y[0] + in_y[1])/2.0 + w = in_x[1] - in_x[0] + h = in_y[1] - in_y[0] x = x*dw w = w*dw y = y*dh @@ -27,25 +31,25 @@ def convert(size, X, Y): def convert_annotation(in_dir, out_dir, image_id, out_id, classes): - in_file = open('%s/%s.xml' % (in_dir, image_id)) - o_file = open(out_dir + '/%s.txt' % out_id, 'w') + in_file = open("%s/%s.xml" % (in_dir, image_id)) + o_file = open(out_dir + "/%s.txt" % out_id, "w") tree = ET.parse(in_file) root = tree.getroot() - size = root.find('imagesize') - h = float(size.find('nrows').text) - w = float(size.find('ncols').text) + size = root.find("imagesize") + h = float(size.find("nrows").text) + w = float(size.find("ncols").text) - for obj in root.iter('object'): + for obj in root.iter("object"): X = [] Y = [] - cls = obj.find('name').text + cls = obj.find("name").text if cls not in classes: logging.debug("%s is not in the selected class" % cls) continue cls_id = classes.index(cls) - for pt in obj.find('polygon').findall('pt'): - X.append(float(pt.find('x').text)) - Y.append(float(pt.find('y').text)) + for pt in obj.find("polygon").findall("pt"): + X.append(float(pt.find("x").text)) + Y.append(float(pt.find("y").text)) if (len(X) < 2 or len(Y) < 2): logging.warning("%s doesn't have sufficient info, ignore" % cls) continue @@ -54,7 +58,7 @@ def convert_annotation(in_dir, out_dir, image_id, out_id, classes): Y = list(set(Y)) Y.sort() bb = convert((w, h), X, Y) - o_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n') + o_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + "\n") def find_output_id(out_img_dir, image_id, suffix): @@ -111,7 +115,7 @@ def main(): elif args.verbosity >= 2: log_level = logging.DEBUG logging.basicConfig(level=log_level, - format='[xmlTotxt: %(levelname)s] %(message)s') + format="[xmlTotxt: %(levelname)s] %(message)s") logging.info(args.classes) @@ -132,21 +136,21 @@ def main(): foutput = join(args.root, "train.txt") if args.delete: - output = open(foutput, 'w') + output = open(foutput, "w") else: - output = open(foutput, 'a') + output = open(foutput, "a") for _img_path in os.listdir(in_img_dir): img_path = os.path.join(in_img_dir, _img_path) - suffix = os.path.basename(_img_path).split('.')[1] + suffix = os.path.basename(_img_path).split(".")[1] logging.debug("Find %s" % img_path) - image_id = os.path.basename(_img_path).split('.')[0] + image_id = os.path.basename(_img_path).split(".")[0] out_id, out_img_path = find_output_id(out_img_dir, image_id, suffix) convert_annotation(in_dir, out_dir, image_id, out_id, args.classes) logging.info("Copy %s to %s" % (img_path, out_img_path)) copyfile(img_path, out_img_path) - output.write(out_img_path + '\n') + output.write(out_img_path + "\n") if __name__ == '__main__': From 1d2a9df6ba736623899319837c9f02250986dd2b Mon Sep 17 00:00:00 2001 From: Tammy Date: Mon, 24 Jul 2017 17:25:42 +0800 Subject: [PATCH 10/41] according to the imports formatting rule, fix the import code block --- utils/xmlTotxt.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/xmlTotxt.py b/utils/xmlTotxt.py index efa0690..ee2f022 100644 --- a/utils/xmlTotxt.py +++ b/utils/xmlTotxt.py @@ -8,13 +8,14 @@ CLASS_i: Class labels defined on LabelMe """ -import os -from os.path import join import argparse import logging -from shutil import copyfile +import os import xml.etree.ElementTree as ET +from os.path import join +from shutil import copyfile + def convert(size, in_x, in_y): dw = 1./size[0] From 177e0442009f7f1958d560f94404ac04b298d1ce Mon Sep 17 00:00:00 2001 From: "Ying-Chun Liu (PaulLiu)" Date: Thu, 27 Jul 2017 13:34:48 +0800 Subject: [PATCH 11/41] Add streaming from USB camera function. Signed-off-by: Ying-Chun Liu (PaulLiu) --- camera.js | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/camera.js b/camera.js index d222332..46b3264 100644 --- a/camera.js +++ b/camera.js @@ -22,6 +22,7 @@ const mqtt = require('mqtt'); const request = require('request'); const spawnsync = require('child_process').spawnSync; const config = require('./config'); +const cv = require('opencv'); const broker = config.brokerHost; const client = mqtt.connect(broker); @@ -36,6 +37,9 @@ const cameraArgs = ['-vf', '-hf', '-o', snapshotFile]; const usbCameraCmd = '/usr/bin/fswebcam'; const usbCameraArgs = ['-r', '1024x768', '--no-banner', '-D', '0.5', snapshotFile]; +var cameraIntervalID = null; +var cameraInterval = 1000.0 / 0.1; // 0.1 fps +var cameraCV = null; function log(m) { client.publish(topicActionLog, m); @@ -88,6 +92,37 @@ client.on('message', (t, m) => { client.publish(topicActionInference, data); } }); + } else if (action == 'stream_usb_start') { + if ((!cameraCV) && (!cameraIntervalID)) { + cameraCV = new cv.VideoCapture(0); + cameraCV.setWidth(1024); + cameraCV.setHeight(768); + cameraIntervalID = setInterval(function() { + cameraCV.read(function(err, im) { + if (err) { + throw err; + } + im.save(snapshotFile); + fs.readFile(snapshotFile, function(err, data) { + if (err) { + log('camera client: cannot get image.'); + } else { + log('camera client: publishing image.'); + client.publish(topicActionInference, data); + } + }); + }); + }, cameraInterval); + } + } else if (action == 'stream_usb_stop') { + if (cameraCV) { + cameraCV.release(); + cameraCV = null; + } + if (cameraIntervalID) { + clearInterval(cameraIntervalID); + cameraIntervalID = null; + } } else { log('camera client: unkown action.'); } From c14cae3202ea3819ba261a473f62ae5b82b2d34c Mon Sep 17 00:00:00 2001 From: "Ying-Chun Liu (PaulLiu)" Date: Thu, 27 Jul 2017 13:38:40 +0800 Subject: [PATCH 12/41] Update README.md for streaming from USB camera Signed-off-by: Ying-Chun Liu (PaulLiu) --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 7290b1d..38ca5a5 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,18 @@ To provide a local image $ mosquitto_pub -h localhost -t berrynet/event/localImage -m ``` +To start streaming from USB camera + +``` +$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_usb_start +``` + +To stop streaming from USB camera + +``` +$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_usb_stop +``` + # Use Your Data To Train The original instruction of retraining YOLOv2 model see [github repository of darknet](https://github.com/AlexeyAB/darknet#how-to-train-to-detect-your-custom-objects) From dcc8e352be83890490bc42b5f75d5912da8297d6 Mon Sep 17 00:00:00 2001 From: "Ying-Chun Liu (PaulLiu)" Date: Sun, 30 Jul 2017 21:34:49 +0800 Subject: [PATCH 13/41] package.json: Add opencv dependecnies. Signed-off-by: Ying-Chun Liu (PaulLiu) --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 5179913..a2f2189 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "mocha": "^3.2.0", "mosca": "^2.2.0", "mqtt": "^2.0.1", + "opencv": "^6.0.0", "pino": "^2.13.0", "prompt": "^1.0.0", "request": "^2.79.0" From 92cf8fcae9f9bf744ac9bac0c0d19e056491187d Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Wed, 16 Aug 2017 18:44:19 +0800 Subject: [PATCH 14/41] Fix model and config files to prevent unexpected update from upstream. --- configure | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index e575acf..ae72c96 100755 --- a/configure +++ b/configure @@ -79,8 +79,8 @@ download_classifier_model() { download_detector_model() { pushd inference/darkflow > /dev/null mkdir bin - wget -O bin/tiny-yolo.weights http://pjreddie.com/media/files/tiny-yolo.weights - wget -O cfg/tiny-yolo.cfg https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/tiny-yolo.cfg + wget -O bin/tiny-yolo.weights https://s3.amazonaws.com/berrynet/models/tinyyolo/tiny-yolo.weights + wget -O cfg/tiny-yolo.cfg https://s3.amazonaws.com/berrynet/models/tinyyolo/tiny-yolo.cfg popd > /dev/null } From ba86003094c1eaec2aafb992c5764881a504f0d9 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Sat, 19 Aug 2017 20:07:46 +0800 Subject: [PATCH 15/41] Fix node-opencv installation issue (close #1). --- configure | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure b/configure index ae72c96..1ce87b7 100755 --- a/configure +++ b/configure @@ -23,7 +23,7 @@ LOG="/tmp/berrynet.log" install_system_dependencies() { sudo apt-get update - sudo apt-get install -y python-dev python-pip python-opencv mongodb libkrb5-dev libzmq3-dev libyaml-dev imagemagick curl fswebcam + sudo apt-get install -y python-dev python-pip python-opencv mongodb libkrb5-dev libzmq3-dev libyaml-dev libopencv-dev imagemagick curl fswebcam sudo service mongodb start sudo -H pip install watchdog cython } @@ -131,7 +131,7 @@ install_gateway() { sudo cp berrynet-manager /usr/local/bin # install npm dependencies pushd $working_dir > /dev/null - sudo npm install + sudo npm install --unsafe-perm popd > /dev/null } diff --git a/package.json b/package.json index a2f2189..653f7ce 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "BerryNet", - "version": "1.0.0", + "version": "2.0.0", "description": "Deep learning gateway on Raspberry Pi", "main": "index.js", "author": "DT42", From 21123683e88f6b22696dd9c1576c67daf786e861 Mon Sep 17 00:00:00 2001 From: "Ying-Chun Liu (PaulLiu)" Date: Wed, 23 Aug 2017 11:17:04 +0800 Subject: [PATCH 16/41] Add libopencv-dev as build dependencies. Add git wget to configure for Ubuntu docker image. Signed-off-by: Ying-Chun Liu (PaulLiu) --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index e575acf..b3e9b5f 100755 --- a/configure +++ b/configure @@ -23,7 +23,7 @@ LOG="/tmp/berrynet.log" install_system_dependencies() { sudo apt-get update - sudo apt-get install -y python-dev python-pip python-opencv mongodb libkrb5-dev libzmq3-dev libyaml-dev imagemagick curl fswebcam + sudo apt-get install -y python-dev python-pip python-opencv mongodb libkrb5-dev libzmq3-dev libyaml-dev imagemagick curl fswebcam wget git libopencv-dev sudo service mongodb start sudo -H pip install watchdog cython } From a321d2709975e013cfe062c7589c6e641694f31c Mon Sep 17 00:00:00 2001 From: "Ying-Chun Liu (PaulLiu)" Date: Fri, 1 Sep 2017 11:49:33 +0800 Subject: [PATCH 17/41] configure: Update npm to latest version. Use latest npm to do installation. Signed-off-by: Ying-Chun Liu (PaulLiu) --- configure | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure b/configure index b3e9b5f..9285681 100755 --- a/configure +++ b/configure @@ -26,6 +26,7 @@ install_system_dependencies() { sudo apt-get install -y python-dev python-pip python-opencv mongodb libkrb5-dev libzmq3-dev libyaml-dev imagemagick curl fswebcam wget git libopencv-dev sudo service mongodb start sudo -H pip install watchdog cython + sudo npm install npm -g } install_optional_dependencies() { @@ -131,7 +132,7 @@ install_gateway() { sudo cp berrynet-manager /usr/local/bin # install npm dependencies pushd $working_dir > /dev/null - sudo npm install + sudo /usr/local/bin/npm install popd > /dev/null } From c2ad8f2954f3b1f611afb1e0b8c616a6bd9a2ec5 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Mon, 4 Sep 2017 18:38:33 +0800 Subject: [PATCH 18/41] Add CONTRIBUTING.md (use DCO instead of CLA). Signed-off-by: Bofu Chen (bafu) --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ef97a78 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,7 @@ +We use [Developer Certificate of Origin](https://developercertificate.org/). + +To use DCO, you only need to add your signature into a Git commit: + +``` +$ git commit -s -m "your commit message." +``` From 4539864a8c0c345619f9622de5cdd566ec443bd4 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Tue, 19 Sep 2017 20:22:25 +0800 Subject: [PATCH 19/41] Respect coding style. --- berrynet-manager | 10 +++++----- client.js | 18 +++++++++--------- inference/agent.js | 32 ++++++++++++++++---------------- inference/classify_server.py | 14 +++++++------- inference/detection_server.py | 12 ++++++------ mail.js | 8 ++++---- 6 files changed, 47 insertions(+), 47 deletions(-) diff --git a/berrynet-manager b/berrynet-manager index 39fce93..aeb46bb 100755 --- a/berrynet-manager +++ b/berrynet-manager @@ -1,19 +1,19 @@ #! /bin/sh # # Copyright 2017 DT42 -# +# # This file is part of BerryNet. -# +# # BerryNet is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. -# +# # BerryNet is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with BerryNet. If not, see . @@ -47,4 +47,4 @@ case $1 in ;; *) help -esac +esac diff --git a/client.js b/client.js index 2228c49..1d57ed2 100644 --- a/client.js +++ b/client.js @@ -1,14 +1,14 @@ -var mqtt = require('mqtt') -//var client = mqtt.connect('mqtt://test.mosquitto.org') -var client = mqtt.connect('mqtt://localhost:1883') +var mqtt = require('mqtt'); +//var client = mqtt.connect('mqtt://test.mosquitto.org'); +var client = mqtt.connect('mqtt://localhost:1883'); client.on('connect', function () { - client.subscribe('presence') - client.publish('presence', 'Hello mqtt') -}) + client.subscribe('presence'); + client.publish('presence', 'Hello mqtt'); +}); client.on('message', function (topic, message) { // message is Buffer - console.log(message.toString()) - client.end() -}) + console.log(message.toString()); + client.end(); +}); diff --git a/inference/agent.js b/inference/agent.js index 4ee9879..d21796a 100644 --- a/inference/agent.js +++ b/inference/agent.js @@ -1,17 +1,17 @@ // Copyright 2017 DT42 -// +// // This file is part of BerryNet. -// +// // BerryNet is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// +// // BerryNet is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -// +// // You should have received a copy of the GNU General Public License // along with BerryNet. If not, see . @@ -54,13 +54,13 @@ client.on('connect', () => { client.on('message', (t, m) => { const size = m.length; const now = moment().format('YYYYMMDD-HHmmss'); - const inference_server_img_dir = __dirname + '/image' - const snapshot = `snapshot-${now}.jpg` - const snapshot_path = path.join(inference_server_img_dir, snapshot) - const donefile_path = snapshot_path + '.done' - const resultfile_path = snapshot_path + '.txt' - const resultdonefile_path = snapshot_path + '.txt.done' - const dashboard_image_path = __dirname + '/../dashboard/www/freeboard/snapshot.jpg' + const inference_server_img_dir = __dirname + '/image'; + const snapshot = `snapshot-${now}.jpg`; + const snapshot_path = path.join(inference_server_img_dir, snapshot); + const donefile_path = snapshot_path + '.done'; + const resultfile_path = snapshot_path + '.txt'; + const resultdonefile_path = snapshot_path + '.txt.done'; + const dashboard_image_path = __dirname + '/../dashboard/www/freeboard/snapshot.jpg'; log(`inference client: on topic ${t}, received ${size} bytes.`); @@ -93,7 +93,7 @@ client.on('message', (t, m) => { fs.readFile(resultfile_path, (err, result) => { if (err) throw err - watcher.close() + watcher.close(); if (inferenceEngine === 'classifier') { fs.writeFile(dashboard_image_path, m, (err, written, buffer) => { @@ -107,15 +107,15 @@ client.on('message', (t, m) => { console.log('Unknown owner ' + inferenceEngine); } - client.publish(topicDashboardInferenceResult, result.toString().replace(/(\n)+/g, '
')) + client.publish(topicDashboardInferenceResult, result.toString().replace(/(\n)+/g, '
')); }) } else { console.log('Detect change of ' + filename + ', but comparing target is ' + snapshot + '.txt.done'); } } else if (eventType == 'rename') { - console.log('watch get rename event for ' + filename) + console.log('watch get rename event for ' + filename); } else { - console.log('watch get unknown event, ' + eventType) + console.log('watch get unknown event, ' + eventType); } - }) + }); }); diff --git a/inference/classify_server.py b/inference/classify_server.py index 405f88e..447f5c3 100644 --- a/inference/classify_server.py +++ b/inference/classify_server.py @@ -1,17 +1,17 @@ # Copyright 2017 DT42 -# +# # This file is part of BerryNet. -# +# # BerryNet is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. -# +# # BerryNet is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with BerryNet. If not, see . @@ -153,7 +153,7 @@ def on_created(self, event): def main(_): """Called by Tensorflow""" - + global sess, threads # Creates graph from saved GraphDef. @@ -172,7 +172,7 @@ def main(_): args=(labels,))) for t in threads: t.start() for t in threads: t.join() - + if __name__ == '__main__': pid = str(os.getpid()) @@ -188,7 +188,7 @@ def main(_): logging("model_dir: ", FLAGS.model_dir) logging("image_dir: ", FLAGS.image_dir) - # workaround the issue that SIGINT cannot be received (fork a child to + # workaround the issue that SIGINT cannot be received (fork a child to # avoid blocking the main process in Thread.join() child_pid = os.fork() if child_pid == 0: diff --git a/inference/detection_server.py b/inference/detection_server.py index 28be01b..7ac0f1d 100644 --- a/inference/detection_server.py +++ b/inference/detection_server.py @@ -1,17 +1,17 @@ # Copyright 2017 DT42 -# +# # This file is part of BerryNet. -# +# # BerryNet is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. -# +# # BerryNet is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with BerryNet. If not, see . @@ -159,7 +159,7 @@ def main(): _logging('config dir: {}'.format(options['model'])) server(tfnet) - + if __name__ == '__main__': logging.basicConfig(filename='/tmp/dlDetector.log', level=logging.DEBUG) @@ -174,7 +174,7 @@ def main(): with open(pidfile, 'w') as f: f.write(pid) - # workaround the issue that SIGINT cannot be received (fork a child to + # workaround the issue that SIGINT cannot be received (fork a child to # avoid blocking the main process in Thread.join() child_pid = os.fork() if child_pid == 0: diff --git a/mail.js b/mail.js index 841d808..20b39fc 100644 --- a/mail.js +++ b/mail.js @@ -1,17 +1,17 @@ // Copyright 2017 DT42 -// +// // This file is part of BerryNet. -// +// // BerryNet is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// +// // BerryNet is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -// +// // You should have received a copy of the GNU General Public License // along with BerryNet. If not, see . From c1e390bff7e92fbee0ab2e406f21ad8f802a3f60 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Wed, 20 Sep 2017 13:05:51 +0800 Subject: [PATCH 20/41] Fix node-opencv installation issue. --- configure | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/configure b/configure index 5d1782d..d422d65 100755 --- a/configure +++ b/configure @@ -26,13 +26,18 @@ install_system_dependencies() { sudo apt-get install -y python-dev python-pip python-opencv mongodb libkrb5-dev libzmq3-dev libyaml-dev imagemagick curl fswebcam wget git libopencv-dev sudo service mongodb start sudo -H pip install watchdog cython - sudo npm install npm -g } install_optional_dependencies() { sudo apt-get install -y mosquitto-clients } +install_nodejs() { + # v6.x is LTS, if you want the latest feature, change to "setup_7.x". + curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash - + sudo apt-get install -y nodejs +} + install_tensorflow() { TENSORFLOW_VERSION="1.0.1" TENSORFLOW_PKGNAME="tensorflow-${TENSORFLOW_VERSION}-cp27-none-linux_armv7l.whl" @@ -85,30 +90,6 @@ download_detector_model() { popd > /dev/null } -install_nodejs() { - # v6.x is LTS, if you want the latest feature, change to "setup_7.x". - curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash - - sudo apt-get install -y nodejs -} - -install_nodejs_legacy() { - # FIXME: Use bash as default ambiguously. - # Raspbian is 32-bit, so we download armv7l instead of arm64. - NODEJS_VERSION="v6.10.2" - NODEJS_OS="linux" - NODEJS_ARCH="armv7l" - #NODEJS_ARCH="x64" - NODEJS_PKGNAME="node-$NODEJS_VERSION-$NODEJS_OS-$NODEJS_ARCH" - if [ ! -e "$NODEJS_PKGNAME" ]; then - if [ ! -e "$PWD/$NODEJS_PKGNAME.tar.xz" ]; then - wget https://nodejs.org/dist/$NODEJS_VERSION/$NODEJS_PKGNAME.tar.xz - fi - tar xJf $NODEJS_PKGNAME.tar.xz - echo "export PATH=$PWD/$NODEJS_PKGNAME/bin:\$PATH" >> $HOME/.bashrc - fi - export PATH=$PWD/$NODEJS_PKGNAME/bin:$PATH -} - install_dashboard() { if [ ! -e "dashboard" ]; then git clone https://github.com/v-i-s-h/PiIoT-dashboard.git dashboard @@ -132,17 +113,17 @@ install_gateway() { sudo cp berrynet-manager /usr/local/bin # install npm dependencies pushd $working_dir > /dev/null - sudo /usr/local/bin/npm install + sudo npm install --unsafe-perm popd > /dev/null } install_system_dependencies 2>&1 | tee -a $LOG install_optional_dependencies 2>&1 | tee -a $LOG +install_nodejs 2>&1 | tee -a $LOG install_tensorflow 2>&1 | tee -a $LOG download_classifier_model 2>&1 | tee -a $LOG install_darkflow 2>&1 | tee -a $LOG download_detector_model 2>&1 | tee -a $LOG -install_nodejs 2>&1 | tee -a $LOG install_dashboard 2>&1 | tee -a $LOG install_systemd_configs 2>&1 | tee -a $LOG install_gateway 2>&1 | tee -a $LOG From 7148c09f9f7dab2c5c3830300f32da264d91e56c Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Wed, 20 Sep 2017 15:09:59 +0800 Subject: [PATCH 21/41] Make OpenCV to access RPi camera via V4L2. --- config/bcm2835-v4l2.conf | 3 +++ configure | 1 + 2 files changed, 4 insertions(+) create mode 100644 config/bcm2835-v4l2.conf diff --git a/config/bcm2835-v4l2.conf b/config/bcm2835-v4l2.conf new file mode 100644 index 0000000..0feba8a --- /dev/null +++ b/config/bcm2835-v4l2.conf @@ -0,0 +1,3 @@ +# BerryNet supports accessing RPi camera access via OpenCV. + +bcm2835_v4l2 diff --git a/configure b/configure index d422d65..c67dee6 100755 --- a/configure +++ b/configure @@ -104,6 +104,7 @@ install_dashboard() { install_systemd_configs() { sudo cp systemd/* /etc/systemd/system + sudo cp config/bcm2835-v4l2.conf /etc/modules-load.d } install_gateway() { From b2062f329b44c96f249868a4a8f596448f33fff6 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Wed, 20 Sep 2017 18:18:51 +0800 Subject: [PATCH 22/41] Add ramfs to speedup I/O. --- configure | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure b/configure index c67dee6..52ef0bb 100755 --- a/configure +++ b/configure @@ -105,6 +105,10 @@ install_dashboard() { install_systemd_configs() { sudo cp systemd/* /etc/systemd/system sudo cp config/bcm2835-v4l2.conf /etc/modules-load.d + # enable ramfs to speedup I/O + echo -e "tmpfs /var/ramfs tmpfs nodev,nosuid,size=50M 0 0" \ + | sudo tee -a /etc/fstab + sudo mount -a } install_gateway() { From d4d1849382bf61ff9935c89c02f49e2db51d8098 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Thu, 21 Sep 2017 19:43:15 +0800 Subject: [PATCH 23/41] Add data collection support. Signed-off-by: Bofu Chen (bafu) --- config.js | 3 + data_collector.js | 126 ++++++++++++++++++++++++++++++++++++++++ utils/local-launcher.js | 6 ++ 3 files changed, 135 insertions(+) create mode 100644 data_collector.js diff --git a/config.js b/config.js index b4bf420..0ba5a07 100644 --- a/config.js +++ b/config.js @@ -48,6 +48,9 @@ config.topicDashboardInferenceResult = padTopicBase('dashboard/inferenceResult') // IP camera config.ipcameraSnapshot = ''; +// data collector configs +config.storageDirPath = '/tmp/berrynet-data'; + // email notification config.senderEmail = 'SENDER_EMAIL'; config.senderPassword = 'SENDER_PASSWORD'; diff --git a/data_collector.js b/data_collector.js new file mode 100644 index 0000000..811edbc --- /dev/null +++ b/data_collector.js @@ -0,0 +1,126 @@ +// Copyright 2017 DT42 +// +// This file is part of BerryNet. +// +// BerryNet is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// BerryNet is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with BerryNet. If not, see . + +'use strict'; + +const fs = require('fs'); +const mqtt = require('mqtt'); +const path = require('path'); +const config = require('./config'); + +const broker = config.brokerHost; +const client = mqtt.connect(broker); +const topicActionLog = config.topicActionLog; +const topicActionInference = config.topicActionInference; +const topicDashboardSnapshot = config.topicDashboardSnapshot; +const topicDashboardInferenceResult = config.topicDashboardInferenceResult; +const storageDirPath = config.storageDirPath; + + +/** + * Log wrapper to publish log message via MQTT and display on console. + * @param {string} m Log message. + */ +function log(m) { + client.publish(topicActionLog, m); + console.log(m); +} + +/** + * Save published MQTT binary data as an image file. + * @param {object} b The binary data published via MQTT. + * @param {string} filepath The file path of the saved image. + */ +function saveBufferToImage(b, filepath) { + fs.writeFile(filepath, b, (e) => { + if (e) + log(`log client: cannot save buffer to image.`); + else + log(`log client: saved buffer to image successfully.`); + }); +} + +/** + * Get time string in ISO format. + * @return {string} Time string. + */ +function getTimeString() { + const d = new Date(); + return d.toISOString(); +} + +/** + * Save snapshot, detection image, and detection JSON to data directory. + * @param {string} topic Subscribed MQTT topic. + * @param {object} message Snapshot binary | 'snapshot.jpg' | detection JSON. + */ +function callbackSaveData(topic, message) { + if (topic == topicActionInference) { + console.log('Get ' + topicActionInference); + + // NOTE: topicActionInference always happens prior other topics. + callbackSaveData.timeString = getTimeString(); + console.log(callbackSaveData.timeString); + const snapshotImage = path.join( + storageDirPath, + callbackSaveData.timeString + '.jpg'); + saveBufferToImage(message, snapshotImage); + } else if (topic == topicDashboardSnapshot) { + console.log('Get ' + topicDashboardSnapshot); + + const detectionImage = path.join( + storageDirPath, + callbackSaveData.timeString + '-detection.jpg'); + /* + fs.readFile(config.snapshot, (err, data) => { + fs.writeFile(detectionImage, data, (e) => { + if (e) + log('Failed to save detection image.'); + }); + }); + */ + fs.createReadStream(config.snapshot) + .pipe(fs.createWriteStream(detectionImage)); + } else if (topic == topicDashboardInferenceResult) { + console.log('Get ' + topicDashboardInferenceResult); + + const detectionJSON = path.join( + storageDirPath, + callbackSaveData.timeString + '-detection.json'); + fs.writeFile(detectionJSON, message, (e) => { + if (e) + log('Failed to save detection JSON.'); + }); + } else { + console.log('Unsubscribed topic ' + topic); + } +} + +fs.mkdir(storageDirPath, (e) => { + if (e) + log('Failed to create data storage dir.'); +}); + +client.on('connect', () => { + client.subscribe(topicActionLog); + client.subscribe(topicActionInference); + client.subscribe(topicDashboardSnapshot); + client.subscribe(topicDashboardInferenceResult); + log(`log client: connected to ${broker} successfully.`); +}); + +client.on('message', callbackSaveData); diff --git a/utils/local-launcher.js b/utils/local-launcher.js index 8460bcf..a9d2cac 100755 --- a/utils/local-launcher.js +++ b/utils/local-launcher.js @@ -26,6 +26,7 @@ const dlDetector = exec('cd inference/darkflow && python detection_server.py', e const inferenceAgent = exec('cd inference && node agent.js', execCallback); const localImageAgent = exec('node localimg.js', execCallback); const webBrowser = exec('DISPLAY=:0 sensible-browser http://localhost:8080/index.html#source=dashboard.json', execCallback); +const dataCollector = exec('node data_collector.js', execCallback); broker.stdout.on('data', function(data) { console.log("[broker] " + data); @@ -55,6 +56,10 @@ inferenceAgent.stdout.on('data', function(data) { console.log('[inferenceAgent] ' + data); }); +dataCollector.stdout.on('data', function(data) { + console.log('[dataCollector] ' + data); +}); + process.on('SIGINT', function() { console.log('Get SIGINT'); broker.kill(); @@ -65,5 +70,6 @@ process.on('SIGINT', function() { //dlClassifier.kill(); dlDetector.kill(); inferenceAgent.kill(); + dataCollector.kill(); process.exit(0); }); From e75b6eb84d56f9b3a7f0819f59fc74fc5a19e638 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Fri, 22 Sep 2017 02:05:07 +0800 Subject: [PATCH 24/41] Add more efficient detection backend. Signed-off-by: Bofu Chen (bafu) --- configure | 38 ++++++ patch/01-detection-backend.patch | 174 ++++++++++++++++++++++++++ patch/02-detection-utilities.patch | 188 +++++++++++++++++++++++++++++ 3 files changed, 400 insertions(+) create mode 100644 patch/01-detection-backend.patch create mode 100644 patch/02-detection-utilities.patch diff --git a/configure b/configure index 52ef0bb..4ce01d4 100755 --- a/configure +++ b/configure @@ -65,6 +65,44 @@ install_darkflow() { popd > /dev/null } +install_darknet() { + # build dependencies + pip install --user PeachPy + pip install --user git+https://github.com/Maratyszcza/confu + + pushd /tmp > /dev/null + git clone https://github.com/ninja-build/ninja.git + pushd ninja > /dev/null + git checkout release + ./configure.py --bootstrap + popd > /dev/null + popd > /dev/null + + sudo apt-get install -y clang + + pushd /tmp > /dev/null + git clone https://github.com/thomaspark-pkj/NNPACK-darknet.git + pushd NNPACK-darknet > /dev/null + $HOME/.local/bin/confu setup + python ./configure.py --backend auto + /tmp/ninja/ninja + sudo cp lib/{libgoogletest-core.a,libnnpack.a,libpthreadpool.a} /usr/lib/ + sudo cp include/nnpack.h /usr/include/ + sudo cp deps/pthreadpool/include/pthreadpool.h /usr/include/ + popd > /dev/null + popd > /dev/null + + # detection backend + pushd inference > /dev/null + git clone https://github.com/thomaspark-pkj/darknet-nnpack.git darknet + pushd darknet > /dev/null + patch -p 1 < ../../patch/01-detection-backend.patch + patch -p 1 < ../../patch/02-detection-utilities.patch + make -j + popd > /dev/null + popd > /dev/null +} + download_classifier_model() { # Inception v3 is default classifier model INCEPTION_PKGNAME=inception_dec_2015.zip diff --git a/patch/01-detection-backend.patch b/patch/01-detection-backend.patch new file mode 100644 index 0000000..f1f1057 --- /dev/null +++ b/patch/01-detection-backend.patch @@ -0,0 +1,174 @@ +diff --git a/Makefile b/Makefile +index 7ba6b25..31950ce 100644 +--- a/Makefile ++++ b/Makefile +@@ -1,6 +1,6 @@ + GPU=0 + CUDNN=0 +-OPENCV=0 ++OPENCV=1 + NNPACK=1 + ARM_NEON=1 + DEBUG=0 +diff --git a/examples/coco.c b/examples/coco.c +index a07906e..170af71 100644 +--- a/examples/coco.c ++++ b/examples/coco.c +@@ -342,7 +342,7 @@ void test_coco(char *cfgfile, char *weightfile, char *filename, float thresh) + printf("%s: Predicted in %f seconds.\n", input, sec(clock()-time)); + get_detection_boxes(l, 1, 1, thresh, probs, boxes, 0); + if (nms) do_nms_sort(boxes, probs, l.side*l.side*l.n, l.classes, nms); +- draw_detections(im, l.side*l.side*l.n, thresh, boxes, probs, coco_classes, alphabet, 80); ++ draw_detections(im, l.side*l.side*l.n, thresh, boxes, probs, coco_classes, alphabet, 80, 0); + save_image(im, "prediction"); + show_image(im, "predictions"); + free_image(im); +diff --git a/examples/detector.c b/examples/detector.c +index 3c4a107..f2de3cc 100644 +--- a/examples/detector.c ++++ b/examples/detector.c +@@ -581,6 +581,9 @@ void test_detector(char *datacfg, char *cfgfile, char *weightfile, char *filenam + list *options = read_data_cfg(datacfg); + char *name_list = option_find_str(options, "names", "data/names.list"); + char **names = get_labels(name_list); ++ char done[256]; ++ FILE *done_signal = NULL; ++ memset(done, 0, 256); + + image **alphabet = load_alphabet(); + network net = parse_network_cfg(cfgfile); +@@ -621,6 +624,7 @@ void test_detector(char *datacfg, char *cfgfile, char *weightfile, char *filenam + //resize_network(&net, sized.w, sized.h); + #endif + layer l = net.layers[net.n-1]; ++ sprintf(done, "%s.txt.done", input); + + box *boxes = calloc(l.w*l.h*l.n, sizeof(box)); + float **probs = calloc(l.w*l.h*l.n, sizeof(float *)); +@@ -634,7 +638,7 @@ void test_detector(char *datacfg, char *cfgfile, char *weightfile, char *filenam + get_region_boxes(l, im.w, im.h, net.w, net.h, thresh, probs, boxes, 0, 0, hier_thresh, 1); + if (nms) do_nms_obj(boxes, probs, l.w*l.h*l.n, l.classes, nms); + //else if (nms) do_nms_sort(boxes, probs, l.w*l.h*l.n, l.classes, nms); +- draw_detections(im, l.w*l.h*l.n, thresh, boxes, probs, names, alphabet, l.classes); ++ draw_detections(im, l.w*l.h*l.n, thresh, boxes, probs, names, alphabet, l.classes, input); + if(outfile){ + save_image(im, outfile); + } +@@ -650,11 +654,13 @@ void test_detector(char *datacfg, char *cfgfile, char *weightfile, char *filenam + cvDestroyAllWindows(); + #endif + } ++ done_signal = fopen(done, "w"); + + free_image(im); + free_image(sized); + free(boxes); + free_ptrs((void **)probs, l.w*l.h*l.n); ++ fclose(done_signal); + if (filename) break; + } + #ifdef NNPACK +diff --git a/examples/yolo.c b/examples/yolo.c +index 5b3fd16..9e74736 100644 +--- a/examples/yolo.c ++++ b/examples/yolo.c +@@ -309,7 +309,7 @@ void test_yolo(char *cfgfile, char *weightfile, char *filename, float thresh) + get_detection_boxes(l, 1, 1, thresh, probs, boxes, 0); + if (nms) do_nms_sort(boxes, probs, l.side*l.side*l.n, l.classes, nms); + //draw_detections(im, l.side*l.side*l.n, thresh, boxes, probs, voc_names, alphabet, 20); +- draw_detections(im, l.side*l.side*l.n, thresh, boxes, probs, voc_names, alphabet, 20); ++ draw_detections(im, l.side*l.side*l.n, thresh, boxes, probs, voc_names, alphabet, 20, 0); + save_image(im, "predictions"); + show_image(im, "predictions"); + +diff --git a/include/darknet.h b/include/darknet.h +index b6b9402..2de7cc0 100644 +--- a/include/darknet.h ++++ b/include/darknet.h +@@ -695,7 +695,7 @@ float box_iou(box a, box b); + void do_nms(box *boxes, float **probs, int total, int classes, float thresh); + data load_all_cifar10(); + box_label *read_boxes(char *filename, int *n); +-void draw_detections(image im, int num, float thresh, box *boxes, float **probs, char **names, image **labels, int classes); ++void draw_detections(image im, int num, float thresh, box *boxes, float **probs, char **names, image **labels, int classes, char* result_file); + + matrix network_predict_data(network net, data test); + image **load_alphabet(); +diff --git a/src/demo.c b/src/demo.c +index 9dc4946..0030d0d 100644 +--- a/src/demo.c ++++ b/src/demo.c +@@ -77,7 +77,7 @@ void *detect_in_thread(void *ptr) + printf("\nFPS:%.1f\n",fps); + printf("Objects:\n\n"); + image display = buff[(buff_index+2) % 3]; +- draw_detections(display, demo_detections, demo_thresh, boxes, probs, demo_names, demo_alphabet, demo_classes); ++ draw_detections(display, demo_detections, demo_thresh, boxes, probs, demo_names, demo_alphabet, demo_classes, 0); + + demo_index = (demo_index + 1)%demo_frame; + running = 0; +diff --git a/src/image.c b/src/image.c +index 83ed382..c1b5b2a 100644 +--- a/src/image.c ++++ b/src/image.c +@@ -190,24 +190,33 @@ image **load_alphabet() + return alphabets; + } + +-void draw_detections(image im, int num, float thresh, box *boxes, float **probs, char **names, image **alphabet, int classes) ++void draw_detections(image im, int num, float thresh, box *boxes, float **probs, char **names, image **alphabet, int classes, char* result_file) + { + int i; +- ++ FILE *predict_result = NULL; ++ char result_txt[256]; ++ memset(result_txt, 0, 256); ++ if (result_file != NULL) { ++ sprintf(result_txt, "%s.txt", result_file); ++ predict_result = fopen(result_txt, "wa"); ++ if (!predict_result) { ++ printf("%s: Predict result file opened error\n", result_txt); ++ return; ++ } ++ } + for(i = 0; i < num; ++i){ + int class = max_index(probs[i], classes); + float prob = probs[i][class]; + if(prob > thresh){ + +- int width = im.h * .006; ++ int width = im.h * .012; + + if(0){ + width = pow(prob, 1./2.)*10+1; + alphabet = 0; + } + +- //printf("%d %s: %.0f%%\n", i, names[class], prob*100); +- printf("%s: %.0f%%\n", names[class], prob*100); ++ printf("%s %.0f%%\n", names[class], prob*100); + int offset = class*123457 % classes; + float red = get_color(2,offset,classes); + float green = get_color(1,offset,classes); +@@ -232,6 +241,12 @@ void draw_detections(image im, int num, float thresh, box *boxes, float **probs, + if(bot > im.h-1) bot = im.h-1; + + draw_box_width(im, left, top, right, bot, width, red, green, blue); ++ // output: label, accuracy, x, y, width, height ++ if (predict_result) ++ fprintf(predict_result, "%s %.2f %d %d %d %d\n", ++ names[class], prob, left, top, right - left, bot - top); ++ printf("%s %.2f %d %d %d %d\n", ++ names[class], prob, left, top, right - left, bot - top); + if (alphabet) { + image label = get_label(alphabet, names[class], (im.h*.03)/10); + draw_label(im, top + width, left, label, rgb); +@@ -239,6 +254,8 @@ void draw_detections(image im, int num, float thresh, box *boxes, float **probs, + } + } + } ++ if (predict_result) ++ fclose(predict_result); + } + + void transpose_image(image im) diff --git a/patch/02-detection-utilities.patch b/patch/02-detection-utilities.patch new file mode 100644 index 0000000..ecac9db --- /dev/null +++ b/patch/02-detection-utilities.patch @@ -0,0 +1,188 @@ +diff --git a/detectord.py b/detectord.py +new file mode 100644 +index 0000000..bf651f2 +--- /dev/null ++++ b/detectord.py +@@ -0,0 +1,149 @@ ++# Copyright 2015 The TensorFlow Authors. All Rights Reserved. ++# Copyright 2016 dt42.io. All Rights Reserved. ++# ++# 09-01-2016 joseph@dt42.io Initial version ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++# ============================================================================= ++ ++"""Simple image classification server with Inception. ++ ++The server monitors image_dir and run inferences on new images added to the ++directory. Every image file should come with another empty file with '.done' ++suffix to signal readiness. Inference result of a image can be read from the ++'.txt' file of that image after '.txt.done' is spotted. ++ ++This is an example the server expects clients to do. Note the order. ++ ++# cp cat.jpg /run/image_dir ++# touch /run/image_dir/cat.jpg.done ++ ++Clients should wait for appearance of 'cat.jpg.txt.done' before getting ++result from 'cat.jpg.txt'. ++""" ++ ++ ++from __future__ import print_function ++import os ++import sys ++import time ++import signal ++import argparse ++import subprocess ++import errno ++from watchdog.observers import Observer ++from watchdog.events import PatternMatchingEventHandler ++ ++ ++def logging(*args): ++ print("[%08.3f]" % time.time(), ' '.join(args), flush=True) ++ ++ ++class EventHandler(PatternMatchingEventHandler): ++ def process(self, event): ++ """ ++ event.event_type ++ 'modified' | 'created' | 'moved' | 'deleted' ++ event.is_directory ++ True | False ++ event.src_path ++ path/to/observed/file ++ """ ++ # the file will be processed there ++ _msg = event.src_path ++ os.remove(_msg) ++ logging(_msg, event.event_type) ++ darknet.stdin.write(_msg.rstrip('.done').encode('utf8') + b'\n') ++ ++ # ignore all other types of events except 'modified' ++ def on_created(self, event): ++ self.process(event) ++ ++ ++def check_pid(pid): ++ try: ++ os.kill(pid, 0) ++ except OSError as err: ++ if err.errno == errno.ESRCH: ++ # ESRCH == No such process ++ return False ++ elif err.errno == errno.EPERM: ++ # EPERM means no permission, and the process exists to deny the ++ # access ++ return True ++ else: ++ raise ++ else: ++ return True ++ ++if __name__ == '__main__': ++ ap = argparse.ArgumentParser() ++ pid = str(os.getpid()) ++ basename = os.path.splitext(os.path.basename(__file__))[0] ++ ap.add_argument('input_dir') ++ ap.add_argument( ++ '-p', '--pid', default='/tmp/%s.pid' % basename, ++ help='pid file path') ++ ap.add_argument( ++ '-fi', '--fifo', default='/tmp/acti_yolo', ++ help='fifo pipe path') ++ ap.add_argument( ++ '-d', '--data', default='cfg/coco.data', ++ help='fifo pipe path') ++ ap.add_argument( ++ '-c', '--config', default='cfg/yolo.cfg', ++ help='fifo pipe path') ++ ap.add_argument( ++ '-w', '--weight', default='yolo.weights', ++ help='fifo pipe path') ++ args = vars(ap.parse_args()) ++ WATCH_DIR = os.path.abspath(args['input_dir']) ++ FIFO_PIPE = os.path.abspath(args['fifo']) ++ data = args['data'] ++ cfg = args['config'] ++ weight = args['weight'] ++ pidfile = args['pid'] ++ ++ if os.path.isfile(pidfile): ++ with open(pidfile) as f: ++ prev_pid = int(f.readline()) ++ if check_pid(prev_pid): ++ logging("{} already exists and process {} is still running, exists.".format( ++ pidfile, prev_pid)) ++ sys.exit(1) ++ else: ++ logging("{} exists but process {} died, clean it up.".format(pidfile, prev_pid)) ++ os.unlink(pidfile) ++ ++ with open(pidfile, 'w') as f: ++ f.write(pid) ++ ++ logging("watch_dir: ", WATCH_DIR) ++ logging("pid: ", pidfile) ++ ++ cmd = ['./darknet', 'detector', 'test', data, cfg, weight, '-out', '/usr/local/berrynet/dashboard/www/freeboard/snapshot'] ++ darknet = subprocess.Popen(cmd, bufsize=0, ++ stdin=subprocess.PIPE, ++ stderr=subprocess.STDOUT) ++ ++ observer = Observer() ++ observer.schedule( ++ EventHandler(['*.jpg.done', '*.png.done']), ++ path=WATCH_DIR, recursive=True) ++ observer.start() ++ try: ++ darknet.wait() ++ except KeyboardInterrupt: ++ logging("Interrupt by user, clean up") ++ os.kill(darknet.pid, signal.SIGKILL) ++ os.unlink(pidfile) +diff --git a/utils/localrun.sh b/utils/localrun.sh +new file mode 100755 +index 0000000..e51b1ff +--- /dev/null ++++ b/utils/localrun.sh +@@ -0,0 +1,27 @@ ++#!/bin/bash ++ ++SNAPSHOT_DIR="$1" ++ ++ ++usage() { ++ echo "Usage: /utils/local_debug.sh SNAPSHOT_DIR" ++ exit 1 ++} ++ ++ ++if [ "$SNAPSHOT_DIR" = "" ]; then ++ usage ++else ++ echo "SNAPSHOT_DIR: $SNAPSHOT_DIR" ++fi ++ ++# Config Makefile to the environment you want, e.g. enable GPU support, etc. ++#make ++ ++#if [ ! -f "yolo.weights" ]; then ++# wget --no-check-certificate https://s3.amazonaws.com/gitlab-ci-data/acti/models/darknet/yolo.weights ++#else ++# echo "model: yolo.weights" ++#fi ++ ++python detectord.py -c tiny-yolo.cfg -w tiny-yolo.weights $SNAPSHOT_DIR From 465221d304afd8f69dce4e38b7b387654b819f3d Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Sat, 23 Sep 2017 22:06:21 +0800 Subject: [PATCH 25/41] Make faster detection backend as systemd service. Signed-off-by: Bofu Chen (bafu) --- berrynet-manager | 4 +- configure | 3 +- inference/agent.js | 25 +--- patch/02-detection-utilities.patch | 188 -------------------------- systemd/detection_fast_server.service | 15 ++ utils/darknet/detectord.py | 149 ++++++++++++++++++++ utils/darknet/utils/localrun.sh | 21 +++ 7 files changed, 193 insertions(+), 212 deletions(-) delete mode 100644 patch/02-detection-utilities.patch create mode 100644 systemd/detection_fast_server.service create mode 100644 utils/darknet/detectord.py create mode 100644 utils/darknet/utils/localrun.sh diff --git a/berrynet-manager b/berrynet-manager index aeb46bb..f2361f5 100755 --- a/berrynet-manager +++ b/berrynet-manager @@ -33,10 +33,10 @@ fi case $1 in start | stop | status) - sudo systemctl $1 detection_server.service agent.service broker.service dashboard.service localimg.service camera.service journal.service cleaner.timer + sudo systemctl $1 detection_fast_server.service agent.service broker.service dashboard.service localimg.service camera.service journal.service cleaner.timer ;; log) - sudo journalctl -x --no-pager -u detection_server.service + sudo journalctl -x --no-pager -u detection_fast_server.service sudo journalctl -x --no-pager -u agent.service sudo journalctl -x --no-pager -u broker.service sudo journalctl -x --no-pager -u dashboard.service diff --git a/configure b/configure index 4ce01d4..a02819e 100755 --- a/configure +++ b/configure @@ -92,12 +92,11 @@ install_darknet() { popd > /dev/null popd > /dev/null - # detection backend + # build detection backend (darknet) pushd inference > /dev/null git clone https://github.com/thomaspark-pkj/darknet-nnpack.git darknet pushd darknet > /dev/null patch -p 1 < ../../patch/01-detection-backend.patch - patch -p 1 < ../../patch/02-detection-utilities.patch make -j popd > /dev/null popd > /dev/null diff --git a/inference/agent.js b/inference/agent.js index d21796a..ce3993c 100644 --- a/inference/agent.js +++ b/inference/agent.js @@ -73,23 +73,8 @@ client.on('message', (t, m) => { // Listen to classifier/detector's result done file. When result done // file (.txt.done) is created, result is available. var watcher = fs.watch(inference_server_img_dir, (eventType, filename) => { - /* Merge inference result and snapshot into single image. */ - if (eventType === 'change') { + if (eventType === 'rename') { if (filename === (snapshot + '.txt.done')) { - /* - fs.open(resultfile_path, 'r', (err, fd) => { - if (err) { - if (err.code === 'ENOENT') { - console.error(resultfile_path + ' does not exist'); - return; - } - throw err; - } - - readMyData(fd); - }); - */ - fs.readFile(resultfile_path, (err, result) => { if (err) throw err @@ -110,12 +95,12 @@ client.on('message', (t, m) => { client.publish(topicDashboardInferenceResult, result.toString().replace(/(\n)+/g, '
')); }) } else { - console.log('Detect change of ' + filename + ', but comparing target is ' + snapshot + '.txt.done'); + console.log('rename event for ' + + filename + + ', but it is not inference result done file.'); } - } else if (eventType == 'rename') { - console.log('watch get rename event for ' + filename); } else { - console.log('watch get unknown event, ' + eventType); + console.log(eventType + ' event for ' + filename + ', ignore it.'); } }); }); diff --git a/patch/02-detection-utilities.patch b/patch/02-detection-utilities.patch deleted file mode 100644 index ecac9db..0000000 --- a/patch/02-detection-utilities.patch +++ /dev/null @@ -1,188 +0,0 @@ -diff --git a/detectord.py b/detectord.py -new file mode 100644 -index 0000000..bf651f2 ---- /dev/null -+++ b/detectord.py -@@ -0,0 +1,149 @@ -+# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -+# Copyright 2016 dt42.io. All Rights Reserved. -+# -+# 09-01-2016 joseph@dt42.io Initial version -+# -+# Licensed under the Apache License, Version 2.0 (the "License"); -+# you may not use this file except in compliance with the License. -+# You may obtain a copy of the License at -+# -+# http://www.apache.org/licenses/LICENSE-2.0 -+# -+# Unless required by applicable law or agreed to in writing, software -+# distributed under the License is distributed on an "AS IS" BASIS, -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+# See the License for the specific language governing permissions and -+# limitations under the License. -+# ============================================================================= -+ -+"""Simple image classification server with Inception. -+ -+The server monitors image_dir and run inferences on new images added to the -+directory. Every image file should come with another empty file with '.done' -+suffix to signal readiness. Inference result of a image can be read from the -+'.txt' file of that image after '.txt.done' is spotted. -+ -+This is an example the server expects clients to do. Note the order. -+ -+# cp cat.jpg /run/image_dir -+# touch /run/image_dir/cat.jpg.done -+ -+Clients should wait for appearance of 'cat.jpg.txt.done' before getting -+result from 'cat.jpg.txt'. -+""" -+ -+ -+from __future__ import print_function -+import os -+import sys -+import time -+import signal -+import argparse -+import subprocess -+import errno -+from watchdog.observers import Observer -+from watchdog.events import PatternMatchingEventHandler -+ -+ -+def logging(*args): -+ print("[%08.3f]" % time.time(), ' '.join(args), flush=True) -+ -+ -+class EventHandler(PatternMatchingEventHandler): -+ def process(self, event): -+ """ -+ event.event_type -+ 'modified' | 'created' | 'moved' | 'deleted' -+ event.is_directory -+ True | False -+ event.src_path -+ path/to/observed/file -+ """ -+ # the file will be processed there -+ _msg = event.src_path -+ os.remove(_msg) -+ logging(_msg, event.event_type) -+ darknet.stdin.write(_msg.rstrip('.done').encode('utf8') + b'\n') -+ -+ # ignore all other types of events except 'modified' -+ def on_created(self, event): -+ self.process(event) -+ -+ -+def check_pid(pid): -+ try: -+ os.kill(pid, 0) -+ except OSError as err: -+ if err.errno == errno.ESRCH: -+ # ESRCH == No such process -+ return False -+ elif err.errno == errno.EPERM: -+ # EPERM means no permission, and the process exists to deny the -+ # access -+ return True -+ else: -+ raise -+ else: -+ return True -+ -+if __name__ == '__main__': -+ ap = argparse.ArgumentParser() -+ pid = str(os.getpid()) -+ basename = os.path.splitext(os.path.basename(__file__))[0] -+ ap.add_argument('input_dir') -+ ap.add_argument( -+ '-p', '--pid', default='/tmp/%s.pid' % basename, -+ help='pid file path') -+ ap.add_argument( -+ '-fi', '--fifo', default='/tmp/acti_yolo', -+ help='fifo pipe path') -+ ap.add_argument( -+ '-d', '--data', default='cfg/coco.data', -+ help='fifo pipe path') -+ ap.add_argument( -+ '-c', '--config', default='cfg/yolo.cfg', -+ help='fifo pipe path') -+ ap.add_argument( -+ '-w', '--weight', default='yolo.weights', -+ help='fifo pipe path') -+ args = vars(ap.parse_args()) -+ WATCH_DIR = os.path.abspath(args['input_dir']) -+ FIFO_PIPE = os.path.abspath(args['fifo']) -+ data = args['data'] -+ cfg = args['config'] -+ weight = args['weight'] -+ pidfile = args['pid'] -+ -+ if os.path.isfile(pidfile): -+ with open(pidfile) as f: -+ prev_pid = int(f.readline()) -+ if check_pid(prev_pid): -+ logging("{} already exists and process {} is still running, exists.".format( -+ pidfile, prev_pid)) -+ sys.exit(1) -+ else: -+ logging("{} exists but process {} died, clean it up.".format(pidfile, prev_pid)) -+ os.unlink(pidfile) -+ -+ with open(pidfile, 'w') as f: -+ f.write(pid) -+ -+ logging("watch_dir: ", WATCH_DIR) -+ logging("pid: ", pidfile) -+ -+ cmd = ['./darknet', 'detector', 'test', data, cfg, weight, '-out', '/usr/local/berrynet/dashboard/www/freeboard/snapshot'] -+ darknet = subprocess.Popen(cmd, bufsize=0, -+ stdin=subprocess.PIPE, -+ stderr=subprocess.STDOUT) -+ -+ observer = Observer() -+ observer.schedule( -+ EventHandler(['*.jpg.done', '*.png.done']), -+ path=WATCH_DIR, recursive=True) -+ observer.start() -+ try: -+ darknet.wait() -+ except KeyboardInterrupt: -+ logging("Interrupt by user, clean up") -+ os.kill(darknet.pid, signal.SIGKILL) -+ os.unlink(pidfile) -diff --git a/utils/localrun.sh b/utils/localrun.sh -new file mode 100755 -index 0000000..e51b1ff ---- /dev/null -+++ b/utils/localrun.sh -@@ -0,0 +1,27 @@ -+#!/bin/bash -+ -+SNAPSHOT_DIR="$1" -+ -+ -+usage() { -+ echo "Usage: /utils/local_debug.sh SNAPSHOT_DIR" -+ exit 1 -+} -+ -+ -+if [ "$SNAPSHOT_DIR" = "" ]; then -+ usage -+else -+ echo "SNAPSHOT_DIR: $SNAPSHOT_DIR" -+fi -+ -+# Config Makefile to the environment you want, e.g. enable GPU support, etc. -+#make -+ -+#if [ ! -f "yolo.weights" ]; then -+# wget --no-check-certificate https://s3.amazonaws.com/gitlab-ci-data/acti/models/darknet/yolo.weights -+#else -+# echo "model: yolo.weights" -+#fi -+ -+python detectord.py -c tiny-yolo.cfg -w tiny-yolo.weights $SNAPSHOT_DIR diff --git a/systemd/detection_fast_server.service b/systemd/detection_fast_server.service new file mode 100644 index 0000000..c6456b6 --- /dev/null +++ b/systemd/detection_fast_server.service @@ -0,0 +1,15 @@ +[Unit] +Description=detection server +After=network.target + +[Service] +Type=simple +WorkingDirectory=/usr/local/berrynet/inference/darknet +PIDFile=/tmp/detection_server.pid +ExecStart=/bin/bash utils/localrun.sh /var/ramfs +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +WantedBy=graphical.target diff --git a/utils/darknet/detectord.py b/utils/darknet/detectord.py new file mode 100644 index 0000000..6729875 --- /dev/null +++ b/utils/darknet/detectord.py @@ -0,0 +1,149 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# Copyright 2016 dt42.io. All Rights Reserved. +# +# 09-01-2016 joseph@dt42.io Initial version +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Simple image classification server with Inception. + +The server monitors image_dir and run inferences on new images added to the +directory. Every image file should come with another empty file with '.done' +suffix to signal readiness. Inference result of a image can be read from the +'.txt' file of that image after '.txt.done' is spotted. + +This is an example the server expects clients to do. Note the order. + +# cp cat.jpg /run/image_dir +# touch /run/image_dir/cat.jpg.done + +Clients should wait for appearance of 'cat.jpg.txt.done' before getting +result from 'cat.jpg.txt'. +""" + + +from __future__ import print_function +import os +import sys +import time +import signal +import argparse +import subprocess +import errno +from watchdog.observers import Observer +from watchdog.events import PatternMatchingEventHandler + + +def logging(*args): + print("[%08.3f]" % time.time(), ' '.join(args)) + + +class EventHandler(PatternMatchingEventHandler): + def process(self, event): + """ + event.event_type + 'modified' | 'created' | 'moved' | 'deleted' + event.is_directory + True | False + event.src_path + path/to/observed/file + """ + # the file will be processed there + _msg = event.src_path + os.remove(_msg) + logging(_msg, event.event_type) + darknet.stdin.write(_msg.rstrip('.done').encode('utf8') + b'\n') + + # ignore all other types of events except 'modified' + def on_created(self, event): + self.process(event) + + +def check_pid(pid): + try: + os.kill(pid, 0) + except OSError as err: + if err.errno == errno.ESRCH: + # ESRCH == No such process + return False + elif err.errno == errno.EPERM: + # EPERM means no permission, and the process exists to deny the + # access + return True + else: + raise + else: + return True + +if __name__ == '__main__': + ap = argparse.ArgumentParser() + pid = str(os.getpid()) + basename = os.path.splitext(os.path.basename(__file__))[0] + ap.add_argument('input_dir') + ap.add_argument( + '-p', '--pid', default='/tmp/%s.pid' % basename, + help='pid file path') + ap.add_argument( + '-fi', '--fifo', default='/tmp/acti_yolo', + help='fifo pipe path') + ap.add_argument( + '-d', '--data', default='cfg/coco.data', + help='fifo pipe path') + ap.add_argument( + '-c', '--config', default='cfg/yolo.cfg', + help='fifo pipe path') + ap.add_argument( + '-w', '--weight', default='yolo.weights', + help='fifo pipe path') + args = vars(ap.parse_args()) + WATCH_DIR = os.path.abspath(args['input_dir']) + FIFO_PIPE = os.path.abspath(args['fifo']) + data = args['data'] + cfg = args['config'] + weight = args['weight'] + pidfile = args['pid'] + + if os.path.isfile(pidfile): + with open(pidfile) as f: + prev_pid = int(f.readline()) + if check_pid(prev_pid): + logging("{} already exists and process {} is still running, exists.".format( + pidfile, prev_pid)) + sys.exit(1) + else: + logging("{} exists but process {} died, clean it up.".format(pidfile, prev_pid)) + os.unlink(pidfile) + + with open(pidfile, 'w') as f: + f.write(pid) + + logging("watch_dir: ", WATCH_DIR) + logging("pid: ", pidfile) + + cmd = ['./darknet', 'detector', 'test', data, cfg, weight, '-out', '/usr/local/berrynet/dashboard/www/freeboard/snapshot'] + darknet = subprocess.Popen(cmd, bufsize=0, + stdin=subprocess.PIPE, + stderr=subprocess.STDOUT) + + observer = Observer() + observer.schedule( + EventHandler(['*.jpg.done', '*.png.done']), + path=WATCH_DIR, recursive=True) + observer.start() + try: + darknet.wait() + except KeyboardInterrupt: + logging("Interrupt by user, clean up") + os.kill(darknet.pid, signal.SIGKILL) + os.unlink(pidfile) diff --git a/utils/darknet/utils/localrun.sh b/utils/darknet/utils/localrun.sh new file mode 100644 index 0000000..f8e6b18 --- /dev/null +++ b/utils/darknet/utils/localrun.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +SNAPSHOT_DIR="$1" +DARKFLOW_DIR="/usr/local/berrynet/inference/darkflow" + +usage() { + echo "Usage: /utils/local_debug.sh SNAPSHOT_DIR" + exit 1 +} + + +if [ "$SNAPSHOT_DIR" = "" ]; then + usage +else + echo "SNAPSHOT_DIR: $SNAPSHOT_DIR" +fi + +python detectord.py \ + -c $DARKFLOW_DIR/cfg/tiny-yolo.cfg \ + -w $DARKFLOW_DIR/bin/tiny-yolo.weights \ + $SNAPSHOT_DIR From f78aa2afb3bb2408c15e271c20e70bea051b20c5 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Sat, 23 Sep 2017 22:10:53 +0800 Subject: [PATCH 26/41] Make data collector as systemd service. Signed-off-by: Bofu Chen (bafu) --- config.js | 2 +- configure | 5 ++++- systemd/data_collector.service | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 systemd/data_collector.service diff --git a/config.js b/config.js index 0ba5a07..ca59ed9 100644 --- a/config.js +++ b/config.js @@ -49,7 +49,7 @@ config.topicDashboardInferenceResult = padTopicBase('dashboard/inferenceResult') config.ipcameraSnapshot = ''; // data collector configs -config.storageDirPath = '/tmp/berrynet-data'; +config.storageDirPath = ''; // email notification config.senderEmail = 'SENDER_EMAIL'; diff --git a/configure b/configure index a02819e..dfab2c6 100755 --- a/configure +++ b/configure @@ -151,7 +151,10 @@ install_systemd_configs() { install_gateway() { local working_dir="/usr/local/berrynet" sudo mkdir -p $working_dir - sudo cp -a broker.js camera.js cleaner.sh config.js dashboard inference journal.js localimg.js mail.js package.json $working_dir + sudo cp -a \ + broker.js camera.js cleaner.sh config.js dashboard data_collector.js \ + inference journal.js localimg.js mail.js package.json \ + $working_dir sudo cp berrynet-manager /usr/local/bin # install npm dependencies pushd $working_dir > /dev/null diff --git a/systemd/data_collector.service b/systemd/data_collector.service new file mode 100644 index 0000000..5dd2664 --- /dev/null +++ b/systemd/data_collector.service @@ -0,0 +1,15 @@ +[Unit] +Description=MQTT client agent for DL inference data collection. +After=network.target + +[Service] +Type=simple +WorkingDirectory=/usr/local/berrynet +PIDFile=/tmp/data_collector.pid +ExecStart=/usr/bin/node data_collector.js +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +WantedBy=graphical.target From 3c7076d2ddf4231ebc39f8b8081aa4a76dec97e5 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Sat, 23 Sep 2017 22:26:45 +0800 Subject: [PATCH 27/41] Fix faster detection backend's config. Signed-off-by: Bofu Chen (bafu) --- configure | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure b/configure index dfab2c6..db3ec2c 100755 --- a/configure +++ b/configure @@ -100,6 +100,10 @@ install_darknet() { make -j popd > /dev/null popd > /dev/null + + cp utils/darknet/detectord.py inference/darknet + mkdir inference/darknet/utils + cp utils/darknet/utils/localrun.sh inference/darknet/utils } download_classifier_model() { From 491d8ef38c427bf7e12a4e9ce61d68f0df3d0fd3 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Sat, 23 Sep 2017 22:47:57 +0800 Subject: [PATCH 28/41] Fix berrynet-manager to launch data collector. --- berrynet-manager | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/berrynet-manager b/berrynet-manager index f2361f5..b8092f2 100755 --- a/berrynet-manager +++ b/berrynet-manager @@ -33,7 +33,7 @@ fi case $1 in start | stop | status) - sudo systemctl $1 detection_fast_server.service agent.service broker.service dashboard.service localimg.service camera.service journal.service cleaner.timer + sudo systemctl $1 detection_fast_server.service agent.service broker.service dashboard.service localimg.service camera.service journal.service cleaner.timer data_collector.service ;; log) sudo journalctl -x --no-pager -u detection_fast_server.service @@ -44,6 +44,7 @@ case $1 in sudo journalctl -x --no-pager -u camera.service sudo journalctl -x --no-pager -u journal.service sudo journalctl -x --no-pager -u cleaner.timer + sudo journalctl -x --no-pager -u data_collector.timer ;; *) help From d52f79895e92ad2c1e9aa7f22c3c7fadf64941c8 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Mon, 25 Sep 2017 15:33:16 +0800 Subject: [PATCH 29/41] Load detection model in the model package. Signed-off-by: Bofu Chen (bafu) --- configure | 4 ++-- utils/darknet/utils/localrun.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configure b/configure index db3ec2c..50c7cfc 100755 --- a/configure +++ b/configure @@ -126,9 +126,9 @@ download_classifier_model() { download_detector_model() { pushd inference/darkflow > /dev/null mkdir bin - wget -O bin/tiny-yolo.weights https://s3.amazonaws.com/berrynet/models/tinyyolo/tiny-yolo.weights - wget -O cfg/tiny-yolo.cfg https://s3.amazonaws.com/berrynet/models/tinyyolo/tiny-yolo.cfg popd > /dev/null + wget -O /tmp/tinyyolo_20170816_all.deb https://s3.amazonaws.com/berrynet/models/tinyyolo/tinyyolo_20170816_all.deb + sudo dpkg -i /tmp/tinyyolo_20170816_all.deb } install_dashboard() { diff --git a/utils/darknet/utils/localrun.sh b/utils/darknet/utils/localrun.sh index f8e6b18..fced5ad 100644 --- a/utils/darknet/utils/localrun.sh +++ b/utils/darknet/utils/localrun.sh @@ -1,7 +1,7 @@ #!/bin/bash SNAPSHOT_DIR="$1" -DARKFLOW_DIR="/usr/local/berrynet/inference/darkflow" +MODEL_DIR="/var/lib/dlmodels/tinyyolo-20170816" usage() { echo "Usage: /utils/local_debug.sh SNAPSHOT_DIR" @@ -16,6 +16,6 @@ else fi python detectord.py \ - -c $DARKFLOW_DIR/cfg/tiny-yolo.cfg \ - -w $DARKFLOW_DIR/bin/tiny-yolo.weights \ + -c $MODEL_DIR/assets/tiny-yolo.cfg \ + -w $MODEL_DIR/tiny-yolo.weights \ $SNAPSHOT_DIR From 850a11dcec2c74e60daf89b9107e4c30db5e3ca1 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Mon, 25 Sep 2017 19:55:15 +0800 Subject: [PATCH 30/41] Fix installation issue. --- configure | 1 + systemd/detection_fast_server.service | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/configure b/configure index 50c7cfc..c991095 100755 --- a/configure +++ b/configure @@ -172,6 +172,7 @@ install_nodejs 2>&1 | tee -a $LOG install_tensorflow 2>&1 | tee -a $LOG download_classifier_model 2>&1 | tee -a $LOG install_darkflow 2>&1 | tee -a $LOG +install_darknet 2>&1 | tee -a $LOG download_detector_model 2>&1 | tee -a $LOG install_dashboard 2>&1 | tee -a $LOG install_systemd_configs 2>&1 | tee -a $LOG diff --git a/systemd/detection_fast_server.service b/systemd/detection_fast_server.service index c6456b6..c3e5d4f 100644 --- a/systemd/detection_fast_server.service +++ b/systemd/detection_fast_server.service @@ -6,7 +6,7 @@ After=network.target Type=simple WorkingDirectory=/usr/local/berrynet/inference/darknet PIDFile=/tmp/detection_server.pid -ExecStart=/bin/bash utils/localrun.sh /var/ramfs +ExecStart=/bin/bash utils/localrun.sh /usr/local/berrynet/inference/image Restart=always RestartSec=10 From 4c1aafdebce1bd4123f869bf0bd0be45d72710ea Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Wed, 27 Sep 2017 18:07:30 +0800 Subject: [PATCH 31/41] Delete intermediate files after dashboard has got the inference result. Remove the regular cleaner to prevent timing issue: an intermediate file is not process yet, but has been deleted because the cleaner's timer is triggered. Signed-off-by: Bofu Chen (bafu) --- berrynet-manager | 13 ++++++++++--- cleaner.sh | 5 ----- config.js | 2 +- configure | 2 +- inference/agent.js | 10 ++++++++++ systemd/cleaner.service | 13 ------------- systemd/cleaner.timer | 9 --------- 7 files changed, 22 insertions(+), 32 deletions(-) delete mode 100644 cleaner.sh delete mode 100644 systemd/cleaner.service delete mode 100644 systemd/cleaner.timer diff --git a/berrynet-manager b/berrynet-manager index b8092f2..a425e68 100755 --- a/berrynet-manager +++ b/berrynet-manager @@ -33,7 +33,15 @@ fi case $1 in start | stop | status) - sudo systemctl $1 detection_fast_server.service agent.service broker.service dashboard.service localimg.service camera.service journal.service cleaner.timer data_collector.service + sudo systemctl $1 \ + detection_fast_server.service \ + agent.service \ + broker.service \ + dashboard.service \ + localimg.service \ + camera.service \ + journal.service \ + data_collector.service ;; log) sudo journalctl -x --no-pager -u detection_fast_server.service @@ -43,8 +51,7 @@ case $1 in sudo journalctl -x --no-pager -u localimg.service sudo journalctl -x --no-pager -u camera.service sudo journalctl -x --no-pager -u journal.service - sudo journalctl -x --no-pager -u cleaner.timer - sudo journalctl -x --no-pager -u data_collector.timer + sudo journalctl -x --no-pager -u data_collector.service ;; *) help diff --git a/cleaner.sh b/cleaner.sh deleted file mode 100644 index 5993dae..0000000 --- a/cleaner.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -TARGETDIR="./inference/image" - -rm -rf $TARGETDIR/snapshot* diff --git a/config.js b/config.js index ca59ed9..0ba5a07 100644 --- a/config.js +++ b/config.js @@ -49,7 +49,7 @@ config.topicDashboardInferenceResult = padTopicBase('dashboard/inferenceResult') config.ipcameraSnapshot = ''; // data collector configs -config.storageDirPath = ''; +config.storageDirPath = '/tmp/berrynet-data'; // email notification config.senderEmail = 'SENDER_EMAIL'; diff --git a/configure b/configure index c991095..627885d 100755 --- a/configure +++ b/configure @@ -156,7 +156,7 @@ install_gateway() { local working_dir="/usr/local/berrynet" sudo mkdir -p $working_dir sudo cp -a \ - broker.js camera.js cleaner.sh config.js dashboard data_collector.js \ + broker.js camera.js config.js dashboard data_collector.js \ inference journal.js localimg.js mail.js package.json \ $working_dir sudo cp berrynet-manager /usr/local/bin diff --git a/inference/agent.js b/inference/agent.js index ce3993c..330a7da 100644 --- a/inference/agent.js +++ b/inference/agent.js @@ -88,6 +88,16 @@ client.on('message', (t, m) => { } else if (inferenceEngine === 'detector') { console.log('Snapshot is created by detector, only notify dashboard to update.'); client.publish(topicDashboardSnapshot, 'snapshot.jpg'); + + // Delete intermediate files. + // + // Note: Data collector will not be affected. It retrieves data from + // * topicActionInference: contains snapshot raw data + // * topicDashboardSnapshot: to copy snapshot with bounding boxes + // * topicDashboardInferenceResult: contains inference result string + fs.unlink(snapshot_path, (e) => {}); + fs.unlink(resultfile_path, (e) => {}); + fs.unlink(resultdonefile_path, (e) => {}); } else { console.log('Unknown owner ' + inferenceEngine); } diff --git a/systemd/cleaner.service b/systemd/cleaner.service deleted file mode 100644 index 6351a0e..0000000 --- a/systemd/cleaner.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Clean snapshots every 5 min -After=network.target - -[Service] -Type=simple -WorkingDirectory=/usr/local/berrynet -PIDFile=/tmp/cleaner.pid -ExecStart=/bin/bash cleaner.sh - -[Install] -WantedBy=multi-user.target -WantedBy=graphical.target diff --git a/systemd/cleaner.timer b/systemd/cleaner.timer deleted file mode 100644 index 2b46c95..0000000 --- a/systemd/cleaner.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Timer to clean snapshots every 5 min. - -[Timer] -OnBootSec=5min -OnUnitActiveSec=5min - -[Install] -WantedBy=timers.target From 2572d414e5f1e9677add3208a188f81872b82f17 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Thu, 28 Sep 2017 14:46:24 +0800 Subject: [PATCH 32/41] Resolve node-opencv memory leak issue. Signed-off-by: Bofu Chen (bafu) --- camera.js | 1 + 1 file changed, 1 insertion(+) diff --git a/camera.js b/camera.js index 46b3264..a0e6563 100644 --- a/camera.js +++ b/camera.js @@ -111,6 +111,7 @@ client.on('message', (t, m) => { client.publish(topicActionInference, data); } }); + im.release(); }); }, cameraInterval); } From 4ec2309285c266617c3be6c6626a254d6be64098 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Fri, 29 Sep 2017 20:27:48 +0800 Subject: [PATCH 33/41] Fix delayed stream issue. --- camera.js | 69 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/camera.js b/camera.js index a0e6563..a17c593 100644 --- a/camera.js +++ b/camera.js @@ -37,9 +37,11 @@ const cameraArgs = ['-vf', '-hf', '-o', snapshotFile]; const usbCameraCmd = '/usr/bin/fswebcam'; const usbCameraArgs = ['-r', '1024x768', '--no-banner', '-D', '0.5', snapshotFile]; +const fps = 30; var cameraIntervalID = null; -var cameraInterval = 1000.0 / 0.1; // 0.1 fps +var cameraInterval = 1000.0 / fps; var cameraCV = null; +var frameCounter = 0; function log(m) { client.publish(topicActionLog, m); @@ -93,37 +95,42 @@ client.on('message', (t, m) => { } }); } else if (action == 'stream_usb_start') { - if ((!cameraCV) && (!cameraIntervalID)) { - cameraCV = new cv.VideoCapture(0); - cameraCV.setWidth(1024); - cameraCV.setHeight(768); - cameraIntervalID = setInterval(function() { - cameraCV.read(function(err, im) { - if (err) { - throw err; - } - im.save(snapshotFile); - fs.readFile(snapshotFile, function(err, data) { - if (err) { - log('camera client: cannot get image.'); - } else { - log('camera client: publishing image.'); - client.publish(topicActionInference, data); - } - }); - im.release(); - }); - }, cameraInterval); - } + if ((!cameraCV) && (!cameraIntervalID)) { + cameraCV = new cv.VideoCapture(0); + cameraCV.setWidth(1024); + cameraCV.setHeight(768); + cameraIntervalID = setInterval(function() { + cameraCV.read(function(err, im) { + if (err) { + throw err; + } + if (frameCounter < fps) { + frameCounter++; + } else { + frameCounter = 0; + im.save(snapshotFile); + fs.readFile(snapshotFile, function(err, data) { + if (err) { + log('camera client: cannot get image.'); + } else { + log('camera client: publishing image.'); + client.publish(topicActionInference, data); + } + }); + } + im.release(); + }); + }, cameraInterval); + } } else if (action == 'stream_usb_stop') { - if (cameraCV) { - cameraCV.release(); - cameraCV = null; - } - if (cameraIntervalID) { - clearInterval(cameraIntervalID); - cameraIntervalID = null; - } + if (cameraCV) { + cameraCV.release(); + cameraCV = null; + } + if (cameraIntervalID) { + clearInterval(cameraIntervalID); + cameraIntervalID = null; + } } else { log('camera client: unkown action.'); } From bcd075e678d293db1ae4b3d9bf3a646b62b914c1 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Fri, 29 Sep 2017 20:31:40 +0800 Subject: [PATCH 34/41] Add RPi3 temperature monitor tool. --- utils/rpi3-temperature.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 utils/rpi3-temperature.sh diff --git a/utils/rpi3-temperature.sh b/utils/rpi3-temperature.sh new file mode 100755 index 0000000..1d0d3ef --- /dev/null +++ b/utils/rpi3-temperature.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Display CPU & GPU temperature for degrees C. +# +# Thanks badfur and yuusou for writing the script. +# https://www.raspberrypi.org/forums/viewtopic.php?t=34994 + +cpuTemp0=$(cat /sys/class/thermal/thermal_zone0/temp) +cpuTemp1=$(($cpuTemp0/1000)) +cpuTemp2=$(($cpuTemp0/100)) +cpuTempM=$(($cpuTemp2 % $cpuTemp1)) + +gpuTemp0=$(/opt/vc/bin/vcgencmd measure_temp) +gpuTemp0=${gpuTemp0//\'/º} +gpuTemp0=${gpuTemp0//temp=/} + +echo CPU Temp: $cpuTemp1"."$cpuTempM"ºC" +echo GPU Temp: $gpuTemp0 From e114b01d4235cd498debe271514945ff9db50a95 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Sat, 30 Sep 2017 13:47:51 +0800 Subject: [PATCH 35/41] Fix delayed stream issue. Signed-off-by: Bofu Chen (bafu) --- camera.js | 14 +++++++++----- config.js | 4 ++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/camera.js b/camera.js index a17c593..02091d6 100644 --- a/camera.js +++ b/camera.js @@ -31,12 +31,16 @@ const topicActionInference = config.topicActionInference; const topicEventCamera = config.topicEventCamera; const cameraURI = config.ipcameraSnapshot; const snapshotFile = '/tmp/snapshot.jpg'; +const snapshotWidth = config.boardcameraImageWidth; +const snapshotHeight = config.boardcameraImageHeight; const cameraCmd = '/usr/bin/raspistill'; const cameraArgs = ['-vf', '-hf', - '-w', '1024', '-h', '768', + '-w', snapshotWidth, + '-h', snapshotHeight, '-o', snapshotFile]; const usbCameraCmd = '/usr/bin/fswebcam'; -const usbCameraArgs = ['-r', '1024x768', '--no-banner', '-D', '0.5', snapshotFile]; +const usbCameraArgs = ['-r', snapshotWidth + 'x' + snapshotHeight, + '--no-banner', '-D', '0.5', snapshotFile]; const fps = 30; var cameraIntervalID = null; var cameraInterval = 1000.0 / fps; @@ -97,14 +101,14 @@ client.on('message', (t, m) => { } else if (action == 'stream_usb_start') { if ((!cameraCV) && (!cameraIntervalID)) { cameraCV = new cv.VideoCapture(0); - cameraCV.setWidth(1024); - cameraCV.setHeight(768); + cameraCV.setWidth(snapshotWidth); + cameraCV.setHeight(snapshotHeight); cameraIntervalID = setInterval(function() { cameraCV.read(function(err, im) { if (err) { throw err; } - if (frameCounter < fps) { + if (frameCounter < fps * 2) { frameCounter++; } else { frameCounter = 0; diff --git a/config.js b/config.js index 0ba5a07..a74b878 100644 --- a/config.js +++ b/config.js @@ -48,6 +48,10 @@ config.topicDashboardInferenceResult = padTopicBase('dashboard/inferenceResult') // IP camera config.ipcameraSnapshot = ''; +// Board camera, e.g. USB and RPi cameras +config.boardcameraImageWidth = 640; +config.boardcameraImageHeight = 480; + // data collector configs config.storageDirPath = '/tmp/berrynet-data'; From d8ae8fb50b4906b092bbe9a03b03ad0cfa35b477 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Sat, 30 Sep 2017 13:52:52 +0800 Subject: [PATCH 36/41] Remove data collector's default storage path. Signed-off-by: Bofu Chen (bafu) --- config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.js b/config.js index a74b878..e6d408e 100644 --- a/config.js +++ b/config.js @@ -53,7 +53,7 @@ config.boardcameraImageWidth = 640; config.boardcameraImageHeight = 480; // data collector configs -config.storageDirPath = '/tmp/berrynet-data'; +config.storageDirPath = ''; // email notification config.senderEmail = 'SENDER_EMAIL'; From a3183ef14a8b9ed1771779906ba382c12c9c95fe Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Sat, 30 Sep 2017 14:52:07 +0800 Subject: [PATCH 37/41] Make RPi and USB cameras to be used via unified name (boardcam). Signed-off-by: Bofu Chen (bafu) --- README.md | 27 +++++++++++++++------------ camera.js | 13 ++++++++++--- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 38ca5a5..1e834d2 100644 --- a/README.md +++ b/README.md @@ -78,22 +78,16 @@ For more details about dashboard configuration (e.g. how to add widgets), please # Provide Image Input -To capture an image via Pi camera - -``` -$ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_picam -``` - To capture an image via configured IP camera ``` $ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_ipcam ``` -To capture an image via USB webcam +To capture an image via board-connected camera (RPi camera or USB webcam) ``` -$ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_usb +$ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_boardcam ``` To provide a local image @@ -102,18 +96,27 @@ To provide a local image $ mosquitto_pub -h localhost -t berrynet/event/localImage -m ``` -To start streaming from USB camera +To start and stop streaming from board-connected camera ``` -$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_usb_start +$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_boardcam_start +$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_boardcam_stop ``` -To stop streaming from USB camera + +# Enable Data Collector + +You might want to store the snapshot and inference results for data analysis. + +To enable data collector, you can set the storage directory path in config.js: ``` -$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_usb_stop +config.storageDirPath = ''; ``` +and restart BerryNet. + + # Use Your Data To Train The original instruction of retraining YOLOv2 model see [github repository of darknet](https://github.com/AlexeyAB/darknet#how-to-train-to-detect-your-custom-objects) diff --git a/camera.js b/camera.js index 02091d6..67ed62f 100644 --- a/camera.js +++ b/camera.js @@ -62,6 +62,13 @@ client.on('message', (t, m) => { const action = m.toString(); if (action == 'snapshot_picam') { + /* NOTE: We use V4L2 to support RPi camera, so RPi camera's usage is + * the same as USB camera. Both RPi and USB cameras are called + * "board camera". + * + * This action is obsoleted and will be removed in the future. + */ + // Take a snapshot from RPi3 camera. The snapshot will be displayed // on dashboard. spawnsync(cameraCmd, cameraArgs); @@ -87,7 +94,7 @@ client.on('message', (t, m) => { } } ); - } else if (action == 'snapshot_usb') { + } else if (action == 'snapshot_boardcam') { // Take a snapshot from USB camera. spawnsync(usbCameraCmd, usbCameraArgs); fs.readFile(snapshotFile, function(err, data) { @@ -98,7 +105,7 @@ client.on('message', (t, m) => { client.publish(topicActionInference, data); } }); - } else if (action == 'stream_usb_start') { + } else if (action == 'stream_boardcam_start') { if ((!cameraCV) && (!cameraIntervalID)) { cameraCV = new cv.VideoCapture(0); cameraCV.setWidth(snapshotWidth); @@ -126,7 +133,7 @@ client.on('message', (t, m) => { }); }, cameraInterval); } - } else if (action == 'stream_usb_stop') { + } else if (action == 'stream_boardcam_stop') { if (cameraCV) { cameraCV.release(); cameraCV = null; From 3c4568243b7061ffcd71ad4e798e9fc7a0af3218 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Sat, 30 Sep 2017 15:09:53 +0800 Subject: [PATCH 38/41] Manually merged Paul's Nest streaming support: camera.js: capture images continuously from cameraURI. This is basically just for Nest IP camera because its API doesn't support real streaming. It can be used with other IP cameras but since other cameras should have the direct way to access the real stream. Thus we don't suggest to use this for ordinary IP camera. Signed-off-by: Ying-Chun Liu (PaulLiu) paulliu@debian.org Signed-off-by: Bofu Chen (bafu) --- README.md | 7 +++++++ camera.js | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/README.md b/README.md index 1e834d2..a0d5658 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,13 @@ $ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_boardcam_start $ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_boardcam_stop ``` +To start and stop streaming from Nest IP camera + +``` +$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_nest_ipcam_start +$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_nest_ipcam_stop +``` + # Enable Data Collector diff --git a/camera.js b/camera.js index 67ed62f..48461db 100644 --- a/camera.js +++ b/camera.js @@ -142,6 +142,27 @@ client.on('message', (t, m) => { clearInterval(cameraIntervalID); cameraIntervalID = null; } + } else if (action == 'stream_nest_ipcam_start') { + if (!cameraIntervalID) { + cameraIntervalID = setInterval(function() { + request.get( + {uri: cameraURI, encoding: null}, + (e, res, body) => { + if (!e && res.statusCode == 200) { + log('camera client: publishing image.'); + client.publish(topicActionInference, body); + } else { + log('camera client: cannot get image.'); + } + } + ); + }, cameraInterval); + } + } else if (action == 'stream_nest_ipcam_stop') { + if (cameraIntervalID) { + clearInterval(cameraIntervalID); + cameraIntervalID = null; + } } else { log('camera client: unkown action.'); } From d8b4081c979e5ddb1795744472f059427a51d68d Mon Sep 17 00:00:00 2001 From: "Ying-Chun Liu (PaulLiu)" Date: Tue, 26 Sep 2017 21:01:57 +0800 Subject: [PATCH 39/41] Add utils for nest IP camera. Signed-off-by: Ying-Chun Liu (PaulLiu) --- utils/nest/nest_get_snapshoturl.sh | 13 +++ utils/nest/nest_get_snapshoturl_by_token.js | 40 +++++++++ utils/nest/nest_get_token.js | 63 ++++++++++++++ utils/nest/nestcamera.md | 96 +++++++++++++++++++++ 4 files changed, 212 insertions(+) create mode 100755 utils/nest/nest_get_snapshoturl.sh create mode 100644 utils/nest/nest_get_snapshoturl_by_token.js create mode 100644 utils/nest/nest_get_token.js create mode 100644 utils/nest/nestcamera.md diff --git a/utils/nest/nest_get_snapshoturl.sh b/utils/nest/nest_get_snapshoturl.sh new file mode 100755 index 0000000..df79afb --- /dev/null +++ b/utils/nest/nest_get_snapshoturl.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +TMPFILE=`mktemp` +NODE='node' + +if [ -x /usr/bin/nodejs ]; then + NODE='/usr/bin/nodejs' +fi +"$NODE" nest_get_token.js | tee "$TMPFILE" + +TOKEN=`cat "$TMPFILE" | grep 'token=' | sed 's/.*token=//'` + +"$NODE" nest_get_snapshoturl_by_token.js "$TOKEN" diff --git a/utils/nest/nest_get_snapshoturl_by_token.js b/utils/nest/nest_get_snapshoturl_by_token.js new file mode 100644 index 0000000..00e04b4 --- /dev/null +++ b/utils/nest/nest_get_snapshoturl_by_token.js @@ -0,0 +1,40 @@ +var Client = require('node-rest-client').Client; + +var ACCESSTOKEN=''; +const NEST_API_URL = 'https://developer-api.nest.com'; + +function usage() { + if (process.argv.length >= 2) { + console.log("Usage: "+process.argv[0]+" "+process.argv[1]+" "); + } else if (process.argv.length >= 1) { + console.log("Usage: "+process.argv[0]+" "); + } else { + console.log("Usage: node nest_get_snapshoturl_by_token.js "); + } +} + +if (process.argv.length <= 2) { + usage(); + process.exit(0); +} + +ACCESSTOKEN=process.argv[2]; + +var client = new Client(); +var args = { + headers: { + "Authorization": 'Bearer ' + ACCESSTOKEN + } +}; + +client.get(NEST_API_URL, args, function (data, response) { + var cameras=data.devices.cameras; + + for (i=0; i { + PINCODE = answer; + + var options = { + "method": "POST", + "hostname": "api.home.nest.com", + "port": null, + "path": "/oauth2/access_token", + "headers": { + "content-type": "application/x-www-form-urlencoded" + } + }; + + var req = http.request(options, function (res) { + var chunks = []; + + res.on("data", function (chunk) { + chunks.push(chunk); + }); + + res.on("end", function () { + var body = Buffer.concat(chunks); + var bodyStr = body.toString(); + + ACCESSTOKEN=JSON.parse(bodyStr); + + var token = ACCESSTOKEN.access_token; + console.log("token="+token); + + }); + }); + + req.write(qs.stringify({ code: PINCODE, + client_id: PRODUCTID, + client_secret: PRODUCTSECRET, + grant_type: 'authorization_code' })); + req.end(); + rl.close(); +}); + diff --git a/utils/nest/nestcamera.md b/utils/nest/nestcamera.md new file mode 100644 index 0000000..4796104 --- /dev/null +++ b/utils/nest/nestcamera.md @@ -0,0 +1,96 @@ +Nest IP Camera libs +======================= + + +Nest IP Camera Introduction +------------------------ +Nest IP Camera is a really closed product. The setup program +for PC are only supported on Windows and MacOS. +If the camera is not connect to your home WiFi, you have to +use Windows or MacOS to let it connect before usage. + +Also the power charger is also important. The camera won't start unless +it connects to its official power charger. So make sure your camera is +connected to WiFi and is running. + + +APIs +------------------------ +There's no way to directly connect to the camera even it is in your local +lan. To access the snapshot from the camera, you have to use their *cloud* +API to obtain it. So basically everything is controlled by their cloud. + + +Old v2/mobile API +------------------------ +This library is inside npm. To install it, just use + +~~~ +npm install nest-api +~~~ + +However, this api is obsolete and no documents can be found. +But it still works partially. The way to use this API is provide your +username (normally e-mail address) and your password. + +~~~ +var NEST_USER='paulliu@dt42.io'; +var NEST_PASSWORD='************'; +var nestApi = new NestApi(NEST_USER, NEST_PASSWORD); + +nestApi.login(function(sessionData) { + console.log(sessionData); + nestApi.get(function(data) { + console.log(data); + nestApi.post({'path':'/v2/mobile/'+sessionData.user+'/quartz/CAMERAID/public_share_enabled', 'body':'true'}, function(data2) { + console.log(data2); + }); + }); +}); +~~~ + +But the post method seems cannot modify the public_shared_enabled flag. +I haven't try all of them. + + +v3 API +------------------------------------------- +This is an OAuth Restful API. You need to register your app/product first. +So please go to https://console.developers.nest.com/products +and register a product. The "support URL" field can be http://localhost because +we are not actually a web app. + +After that, you'll get 3 credentials. + + 1. Product ID + 2. Product Secret + 3. Authorization URL + +First, generate proper unique and secure STATE parameter and replace the +STATE in Authorization URL, show that URL to the user. + +The user will use that URL in the browser and get a PINCODE to you. + +Then use the PINCODE to get a token. + +By using that token, you can use all of the Rest APIs. + +We wrote 3 small scripts to show how to do this. +First, please replace the credentials provided from Nest into nest_get_token.js + +Each time you run nest_get_token.js you'll be shown an URL and waiting you +to enter the PINCODE. You have to use your browser to open that URL and +obtain the PINCODE for the program to continue. The program will later +show you the token after you input the PINCODE. + +And then you can run nest_get_snapshoturl_by_token.js and provide the token +obtain from the above to get the snapshoturl. + +We also wrote a small script for you to do that. Just run +nest_get_snapshoturl.sh and it will call the two scripts from the above and +give you the snapshoturl of Nest IP camera. + + +Streaming +--------------- +Currently there's no way to obtain the streaming from Nest IP camera. From 0586f73f575d47f4bc17add30ad2c514a6300574 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Mon, 9 Oct 2017 18:14:46 +0800 Subject: [PATCH 40/41] Convert data collector's input text data format to JSON. Signed-off-by: Bofu Chen (bafu) --- config.js | 13 +++++++------ data_collector.js | 8 ++++---- inference/agent.js | 31 ++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/config.js b/config.js index da51f70..6845e16 100644 --- a/config.js +++ b/config.js @@ -45,6 +45,7 @@ config.topicNotifyLINE = padTopicBase('notify/line'); config.topicDashboardLog = padTopicBase('dashboard/log'); config.topicDashboardSnapshot = padTopicBase('dashboard/snapshot'); config.topicDashboardInferenceResult = padTopicBase('dashboard/inferenceResult'); +config.topicJSONInferenceResult = padTopicBase('data/jsonInferenceResult'); // IP camera config.ipcameraSnapshot = ''; @@ -57,9 +58,9 @@ config.boardcameraImageHeight = 480; config.storageDirPath = ''; // email notification -config.senderEmail = 'SENDER_EMAIL'; -config.senderPassword = 'SENDER_PASSWORD'; -config.receiverEmail = 'RECEIVER_EMAIL'; +config.senderEmail = ''; +config.senderPassword = ''; +config.receiverEmail = ''; // for compatibility config.sender_email = config.senderEmail; @@ -67,9 +68,9 @@ config.sender_password = config.senderPassword; config.receiver_email = config.receiverEmail; // Authentication and channel information for LINE -config.LINETargetUserID = 'LINE_TARGET_USER_ID'; -config.LINEChannelSecret = 'LINE_CHANNEL_SECRET'; -config.LINEChannelAccessToken = 'LINE_CHANNEL_ACCESS_TOKEN'; +config.LINETargetUserID = ''; +config.LINEChannelSecret = ''; +config.LINEChannelAccessToken = ''; // make config importable module.exports = config; diff --git a/data_collector.js b/data_collector.js index 811edbc..f159fdc 100644 --- a/data_collector.js +++ b/data_collector.js @@ -27,7 +27,7 @@ const client = mqtt.connect(broker); const topicActionLog = config.topicActionLog; const topicActionInference = config.topicActionInference; const topicDashboardSnapshot = config.topicDashboardSnapshot; -const topicDashboardInferenceResult = config.topicDashboardInferenceResult; +const topicJSONInferenceResult = config.topicJSONInferenceResult; const storageDirPath = config.storageDirPath; @@ -95,8 +95,8 @@ function callbackSaveData(topic, message) { */ fs.createReadStream(config.snapshot) .pipe(fs.createWriteStream(detectionImage)); - } else if (topic == topicDashboardInferenceResult) { - console.log('Get ' + topicDashboardInferenceResult); + } else if (topic == topicJSONInferenceResult) { + console.log('Get ' + topicJSONInferenceResult); const detectionJSON = path.join( storageDirPath, @@ -119,7 +119,7 @@ client.on('connect', () => { client.subscribe(topicActionLog); client.subscribe(topicActionInference); client.subscribe(topicDashboardSnapshot); - client.subscribe(topicDashboardInferenceResult); + client.subscribe(topicJSONInferenceResult); log(`log client: connected to ${broker} successfully.`); }); diff --git a/inference/agent.js b/inference/agent.js index e5982ce..6f85c73 100644 --- a/inference/agent.js +++ b/inference/agent.js @@ -30,6 +30,7 @@ const topicActionLog = config.topicActionLog; const topicActionInference = config.topicActionInference; const topicDashboardSnapshot = config.topicDashboardSnapshot; const topicDashboardInferenceResult = config.topicDashboardInferenceResult; +const topicJSONInferenceResult = config.topicJSONInferenceResult; const topicNotifyLINE = config.topicNotifyLINE; const inferenceEngine = config.inferenceEngine; @@ -47,6 +48,29 @@ function saveBufferToImage(b, filepath) { }); } +const parseDarknet = function(str) { + let [label, confidence, x, y, width, height] = str.split(' '); + let result = { + label: label, + confidence: parseFloat(confidence), + top: parseInt(y), + bottom: parseInt(y) + parseInt(height), + left: parseInt(x), + right: parseInt(x) + parseInt(width) + }; + return result +} + +const darknetToJSON = function(data) { + let dataStrList = data.toString().replace(/\n$/, '').split('\n'); + let jsonResult = []; + for (let i in dataStrList) { + let item = `${dataStrList[i]}`; + jsonResult.push(parseDarknet(item)); + } + return jsonResult; +}; + client.on('connect', () => { client.subscribe(topicActionInference); log(`inference client: connected to ${broker} successfully.`); @@ -86,9 +110,15 @@ client.on('message', (t, m) => { console.log('Written snapshot to dashboard image directory: ' + dashboard_image_path); client.publish(topicDashboardSnapshot, 'snapshot.jpg'); }) + client.publish(topicDashboardInferenceResult, + result.toString().replace(/(\n)+/g, '
')); } else if (inferenceEngine === 'detector') { console.log('Snapshot is created by detector, only notify dashboard to update.'); client.publish(topicDashboardSnapshot, 'snapshot.jpg'); + client.publish(topicDashboardInferenceResult, + result.toString().replace(/(\n)+/g, '
')); + client.publish(topicJSONInferenceResult, + JSON.stringify(darknetToJSON(result))); // Delete intermediate files. // @@ -104,7 +134,6 @@ client.on('message', (t, m) => { } client.publish(topicNotifyLINE, dashboard_image_path); - client.publish(topicDashboardInferenceResult, result.toString().replace(/(\n)+/g, '
')) }) } else { console.log('rename event for ' + From 1ae71034fd333668ebdd307dea7462454dffc2e5 Mon Sep 17 00:00:00 2001 From: "Bofu Chen (bafu)" Date: Mon, 9 Oct 2017 18:50:56 +0800 Subject: [PATCH 41/41] Fix inference result parsing issue. In Darknet's result format, an inference result consists of elements seperated by spaces. But a label might also consists of multiple words seperated by spaces, which will cause a parsing failure. Signed-off-by: Bofu Chen (bafu) --- inference/agent.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/inference/agent.js b/inference/agent.js index 6f85c73..a1eabc1 100644 --- a/inference/agent.js +++ b/inference/agent.js @@ -49,7 +49,10 @@ function saveBufferToImage(b, filepath) { } const parseDarknet = function(str) { - let [label, confidence, x, y, width, height] = str.split(' '); + const elements = str.split(' '); + // label might consists of multiple words, e.g. cell phone. + const label = elements.slice(0, elements.length - 5).join(' '); + let [confidence, x, y, width, height] = elements.slice(elements.length - 5); let result = { label: label, confidence: parseFloat(confidence),