From 2e997b53c20a8636a9977373789decbef7f501f2 Mon Sep 17 00:00:00 2001 From: vinjn Date: Tue, 17 Oct 2017 22:51:31 +0800 Subject: [PATCH] src/IFaceTracker re-org Add OpenFaceTracker.* --- src/BaseFaceTracker.cpp | 37 ++ .../IFaceTracker.h => BaseFaceTracker.h} | 10 +- src/FaceVFXApp.cpp | 12 +- src/IFaceTracker/IFaceTracker.cpp | 30 -- .../jason_saragih_tracker/ciFaceTracker.cpp | 362 ----------------- .../jason_saragih_tracker/ciFaceTracker.h | 135 ------- src/JasonFaceTracker.cpp | 365 ++++++++++++++++++ src/JasonFaceTracker.h | 131 +++++++ src/OpenFaceTracker.cpp | 49 +++ src/OpenFaceTracker.h | 23 ++ 10 files changed, 616 insertions(+), 538 deletions(-) create mode 100644 src/BaseFaceTracker.cpp rename src/{IFaceTracker/IFaceTracker.h => BaseFaceTracker.h} (73%) delete mode 100644 src/IFaceTracker/IFaceTracker.cpp delete mode 100644 src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.cpp delete mode 100644 src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.h create mode 100644 src/JasonFaceTracker.cpp create mode 100644 src/JasonFaceTracker.h create mode 100644 src/OpenFaceTracker.cpp create mode 100644 src/OpenFaceTracker.h diff --git a/src/BaseFaceTracker.cpp b/src/BaseFaceTracker.cpp new file mode 100644 index 0000000..38efd5d --- /dev/null +++ b/src/BaseFaceTracker.cpp @@ -0,0 +1,37 @@ +#include "BaseFaceTracker.h" +#include "JasonFaceTracker.h" +#include "OpenFaceTracker.h" + +namespace jing +{ + using namespace std; + using namespace ci; + + BaseFaceTrackerRef BaseFaceTracker::create(Option option) + { + BaseFaceTracker* tracker = nullptr; + + if (option.method == "JasonFaceTracker") + { + auto ciTracker = new ciFaceTracker; + ciTracker->setRescale(option.scale); + + tracker = ciTracker; + } + else if (option.method == "OpenFaceTracker") + { + auto openFaceTracker = new OpenFaceTracker; + + tracker = openFaceTracker; + } + else + { + throw ci::Exception("Unknown method"); + } + + auto trackerRef = BaseFaceTrackerRef(tracker); + trackerRef->setup(); + + return trackerRef; + } +} diff --git a/src/IFaceTracker/IFaceTracker.h b/src/BaseFaceTracker.h similarity index 73% rename from src/IFaceTracker/IFaceTracker.h rename to src/BaseFaceTracker.h index 2cdf3cd..9f5d8e5 100644 --- a/src/IFaceTracker/IFaceTracker.h +++ b/src/BaseFaceTracker.h @@ -4,15 +4,15 @@ #include #include "cinder/TriMesh.h" -namespace ft +namespace jing { - typedef std::shared_ptr FaceTrackerRef; + typedef std::shared_ptr BaseFaceTrackerRef; struct Option { Option() { - method = "jason_saragih_tracker"; + method = "JasonFaceTracker"; scale = 1.0f; } @@ -20,9 +20,9 @@ namespace ft float scale; }; - struct IFaceTracker + struct BaseFaceTracker { - static FaceTrackerRef create(Option option = Option()); + static BaseFaceTrackerRef create(Option option = Option()); virtual bool update(const ci::Surface& surface) = 0; virtual void reset() = 0; virtual int size() const = 0; diff --git a/src/FaceVFXApp.cpp b/src/FaceVFXApp.cpp index 8ff6f8a..910147a 100644 --- a/src/FaceVFXApp.cpp +++ b/src/FaceVFXApp.cpp @@ -21,7 +21,7 @@ #include "cinder/qtime/QuickTimeGl.h" #endif -#include "IFaceTracker/IFaceTracker.h" +#include "BaseFaceTracker.h" #include "Cinder-VNM/include/TextureHelper.h" #include "Cinder-VNM/include/CaptureHelper.h" @@ -121,8 +121,8 @@ class FaceOff : public App TriMesh mFaceMesh; - ft::FaceTrackerRef mOnlineTracker; - ft::FaceTrackerRef mOfflineTracker; + jing::BaseFaceTrackerRef mOnlineTracker; + jing::BaseFaceTrackerRef mOfflineTracker; gl::TextureRef mPhotoTex; #ifdef QUICKTIME_ENABLED @@ -186,10 +186,10 @@ void FaceOff::updateClone() void FaceOff::trackerThreadFn() { - ft::Option option; + jing::Option option; option.scale = 0.5f; - mOfflineTracker = ft::IFaceTracker::create(); - mOnlineTracker = ft::IFaceTracker::create(option); + mOfflineTracker = jing::BaseFaceTracker::create(); + mOnlineTracker = jing::BaseFaceTracker::create(option); bool shouldInitFaceMesh = false; diff --git a/src/IFaceTracker/IFaceTracker.cpp b/src/IFaceTracker/IFaceTracker.cpp deleted file mode 100644 index 598117a..0000000 --- a/src/IFaceTracker/IFaceTracker.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "IFaceTracker.h" -#include "jason_saragih_tracker/ciFacetracker.h" - -namespace ft -{ - using namespace std; - using namespace ci; - - FaceTrackerRef IFaceTracker::create(Option option) - { - IFaceTracker* tracker = nullptr; - - if (option.method == "jason_saragih_tracker") - { - auto ciTracker = new ciFaceTracker; - ciTracker->setRescale(option.scale); - - tracker = ciTracker; - } - else - { - throw ci::Exception("Unknown method"); - } - - auto trackerRef = FaceTrackerRef(tracker); - trackerRef->setup(); - - return trackerRef; - } -} diff --git a/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.cpp b/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.cpp deleted file mode 100644 index 89a6d36..0000000 --- a/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.cpp +++ /dev/null @@ -1,362 +0,0 @@ -#include "ciFaceTracker.h" -#include "cinder/app/App.h" -#include "cinder/Utilities.h" -#include "cinder/gl/gl.h" -#include "cinder/gl/gl.h" -#include "CinderOpenCV.h" - -#include - -using namespace ci; -using namespace std; -using namespace cv; -using namespace FACETRACKER; - -#define it at -#define db at - -// can be compiled with OpenMP for even faster threaded execution - -vector consecutive(int start, int end) { - int n = end - start; - vector result(n); - for (int i = 0; i < n; i++) { - result[i] = start + i; - } - return result; -} - -PolyLine3 ciFaceTracker::getFeature(Feature feature, const vector& points) const { - PolyLine3 polyline; - if (!mIsFailed) { - vector indices = getFeatureIndices(feature); - for (int i = 0; i < indices.size(); i++) { - int cur = indices[i]; - if (mUseInvisible || getVisibility(cur)) { - polyline.push_back(points[cur]); - } - } - } - return polyline; -} - -vector ciFaceTracker::getFeatureIndices(Feature feature) { - static int innerMouth[] = { 48, 60, 61, 62, 54, 63, 64, 65 }; - switch (feature) { - case LEFT_JAW: return consecutive(0, 9); - case RIGHT_JAW: return consecutive(8, 17); - case JAW: return consecutive(0, 17); - case LEFT_EYEBROW: return consecutive(17, 22); - case RIGHT_EYEBROW: return consecutive(22, 27); - case LEFT_EYE: return consecutive(36, 42); - case RIGHT_EYE: return consecutive(42, 48); - case OUTER_MOUTH: return consecutive(48, 60); - case INNER_MOUTH: - return vector(innerMouth, innerMouth + 8); - default: - return consecutive(0, 0); - } -} - -ciFaceTracker::ciFaceTracker() - :mRescale(1) - , mIterations(10) // [1-25] 1 is fast and inaccurate, 25 is slow and accurate - , mClamp(3) // [0-4] 1 gives a very loose fit, 4 gives a very tight fit - , mTolerance(.01) // [.01-1] match tolerance - , mAttempts(1) // [1-4] 1 is fast and may not find faces, 4 is slow but will find faces - , mIsFailed(true) - , mFailCheck(true) // check for whether the tracking failed - , mFrameSkip(-1) // how often to skip frames - , mUseInvisible(true) -{ -} - -void ciFaceTracker::setup() { - wSize1.resize(1); - wSize1[0] = 7; - - wSize2.resize(3); - wSize2[0] = 11; - wSize2[1] = 9; - wSize2[2] = 7; - - string ftFile = app::getAssetPath("face2.tracker").string(); - string triFile = app::getAssetPath("face.tri").string(); - string conFile = app::getAssetPath("face.con").string(); - - mTracker.Load(ftFile.c_str()); - mTri = IO::LoadTri(triFile.c_str()); - mCon = IO::LoadCon(conFile.c_str()); // not being used right now -} -bool ciFaceTracker::update(const Surface& surface) { - Mat image = toOcv(surface); - - if (mRescale == 1) { - mIm = image; - } - else { - resize(image, mIm, cv::Size(mRescale * image.cols, mRescale * image.rows)); - } - - int nChannels = mIm.channels(); - if (nChannels == 3) - cvtColor(mIm, mGray, CV_RGB2GRAY); - else if (nChannels == 4) - cvtColor(mIm, mGray, CV_RGBA2GRAY); - else - mGray = mIm; - mImgSize = { image.cols, image.rows }; - - bool tryAgain = true; - for (int i = 0; tryAgain && i < mAttempts; i++) { - vector wSize; - if (mIsFailed) { - wSize = wSize2; - } - else { - wSize = wSize1; - } - - if (mTracker.Track(mGray, wSize, mFrameSkip, mIterations, mClamp, mTolerance, mFailCheck) == 0) { - mCurrentView = mTracker._clm.GetViewIdx(); - mIsFailed = false; - tryAgain = false; - updateObjectPoints(); - } - else { - mTracker.FrameReset(); - mIsFailed = true; - } - } - return !mIsFailed; -} - -void ciFaceTracker::draw() const{ - if (mIsFailed) { - return; - } - - gl::draw(getImageMesh()); - - int n = size(); - for (int i = 0; i < n; i++){ - if (getVisibility(i)) { - gl::drawString(toString(i), { getImagePoint(i).x, getImagePoint(i).y }); - } - } -} - -void ciFaceTracker::reset() { - mTracker.FrameReset(); -} - -int ciFaceTracker::size() const { - return mTracker._shape.rows / 2; -} - -bool ciFaceTracker::getFound() const { - return !mIsFailed; -} - -bool ciFaceTracker::getVisibility(int i) const { - if (mIsFailed) { - return false; - } - const Mat& visi = mTracker._clm._visi[mCurrentView]; - return (visi.it(i, 0) != 0); -} - -vector ciFaceTracker::getImagePoints() const { - int n = size(); - vector imagePoints(n); - for (int i = 0; i < n; i++) { - imagePoints[i] = getImagePoint(i); - } - return imagePoints; -} - -vector ciFaceTracker::getObjectPoints() const { - int n = size(); - vector objectPoints(n); - for (int i = 0; i < n; i++) { - objectPoints[i] = getObjectPoint(i); - } - return objectPoints; -} - -vector ciFaceTracker::getMeanObjectPoints() const { - int n = size(); - vector meanObjectPoints(n); - for (int i = 0; i < n; i++) { - meanObjectPoints[i] = getMeanObjectPoint(i); - } - return meanObjectPoints; -} - -vec3 ciFaceTracker::getImagePoint(int i) const { - if (mIsFailed) { - return vec3(); - } - const Mat& shape = mTracker._shape; - int n = shape.rows / 2; - return vec3(shape.db(i, 0), shape.db(i + n, 0), 0.0f) / mRescale; -} - -vec3 ciFaceTracker::getObjectPoint(int i) const { - if (mIsFailed) { - return vec3(); - } - int n = mObjectPoints.rows / 3; - return vec3(mObjectPoints.db(i, 0), mObjectPoints.db(i + n, 0), mObjectPoints.db(i + n + n, 0)); -} - -vec3 ciFaceTracker::getMeanObjectPoint(int i) const { - const Mat& mean = mTracker._clm._pdm._M; - int n = mean.rows / 3; - return vec3(mean.db(i, 0), mean.db(i + n, 0), mean.db(i + n + n, 0)); -} - -TriMesh ciFaceTracker::getImageMesh() const{ - return getMesh(getImagePoints()); -} - -TriMesh ciFaceTracker::getObjectMesh() const { - return getMesh(getObjectPoints()); -} - -TriMesh ciFaceTracker::getMeanObjectMesh() const { - return getMesh(getMeanObjectPoints()); -} - -const Mat& ciFaceTracker::getObjectPointsMat() const { - return mObjectPoints; -} - -vec2 ciFaceTracker::getPosition() const { - const Mat& pose = mTracker._clm._pglobl; - return vec2(pose.db(4, 0), pose.db(5, 0)) / mRescale; -} - -// multiply by ~20-23 to get pixel units (+/-20 units in the x axis, +23/-18 on the y axis) -float ciFaceTracker::getScale() const { - const Mat& pose = mTracker._clm._pglobl; - return pose.db(0, 0) / mRescale; -} - -vec3 ciFaceTracker::getOrientation() const { - const Mat& pose = mTracker._clm._pglobl; - vec3 euler(pose.db(1, 0), pose.db(2, 0), pose.db(3, 0)); - return euler; -} - -mat4 ciFaceTracker::getRotationMatrix() const { - vec3 euler = getOrientation(); - mat4 matrix = glm::eulerAngleYXZ(euler.y, euler.x, euler.z); - return matrix; -} - -ciFaceTracker::Direction ciFaceTracker::getDirection() const { - if (mIsFailed) { - return FACING_UNKNOWN; - } - switch (mCurrentView) { - case 0: return FACING_FORWARD; - case 1: return FACING_LEFT; - case 2: return FACING_RIGHT; - default: return FACING_UNKNOWN;; - } -} - -PolyLine3 ciFaceTracker::getImageFeature(Feature feature) const { - return getFeature(feature, getImagePoints()); -} - -PolyLine3 ciFaceTracker::getObjectFeature(Feature feature) const { - return getFeature(feature, getObjectPoints()); -} - -float ciFaceTracker::getGesture(Gesture gesture) const { - if (mIsFailed) { - return 0; - } - int start = 0, end = 0; - switch (gesture) { - // left to right of mouth - case MOUTH_WIDTH: start = 48; end = 54; break; - // top to bottom of inner mouth - case MOUTH_HEIGHT: start = 61; end = 64; break; - // center of the eye to middle of eyebrow - case LEFT_EYEBROW_HEIGHT: start = 38; end = 20; break; - // center of the eye to middle of eyebrow - case RIGHT_EYEBROW_HEIGHT: start = 43; end = 23; break; - // upper inner eye to lower outer eye - case LEFT_EYE_OPENNESS: start = 38; end = 41; break; - // upper inner eye to lower outer eye - case RIGHT_EYE_OPENNESS: start = 43; end = 46; break; - // nose center to chin center - case JAW_OPENNESS: start = 33; end = 8; break; - // left side of nose to right side of nose - case NOSTRIL_FLARE: start = 31; end = 35; break; - } - - return glm::length(getObjectPoint(start) - getObjectPoint(end)); -} - -void ciFaceTracker::setRescale(float rescale) { - mRescale = rescale; -} - -void ciFaceTracker::setIterations(int iterations) { - mIterations = iterations; -} - -void ciFaceTracker::setClamp(float clamp) { - mClamp = clamp; -} - -void ciFaceTracker::setTolerance(float tolerance) { - mTolerance = tolerance; -} - -void ciFaceTracker::setAttempts(int attempts) { - mAttempts = attempts; -} - -void ciFaceTracker::setUseInvisible(bool useInvisible) { - mUseInvisible = useInvisible; -} - -void ciFaceTracker::updateObjectPoints() { - const Mat& mean = mTracker._clm._pdm._M; - const Mat& variation = mTracker._clm._pdm._V; - const Mat& weights = mTracker._clm._plocal; - mObjectPoints = mean + variation * weights; -} - -void ciFaceTracker::addTriangleIndices(TriMesh& mesh) const { - for (int i = 0; i < mTri.rows; i++) { - int i0 = mTri.it(i, 0), i1 = mTri.it(i, 1), i2 = mTri.it(i, 2); - bool visible = getVisibility(i0) && getVisibility(i1) && getVisibility(i2); - if (mUseInvisible || visible) { - mesh.appendTriangle(i0, i1, i2); - } - } -} - -TriMesh ciFaceTracker::getMesh(const vector& points) const { - TriMesh mesh; - if (!mIsFailed) { - int n = size(); - for (int i = 0; i < n; i++) { - mesh.appendPosition(points[i]); - mesh.appendTexCoord({ getImagePoint(i).x / mImgSize.x, getImagePoint(i).y / mImgSize.y }); - } - addTriangleIndices(mesh); - } - return mesh; -} - -const vec2 ciFaceTracker::getImageSize() const -{ - return mImgSize; -} diff --git a/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.h b/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.h deleted file mode 100644 index 5e606b0..0000000 --- a/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.h +++ /dev/null @@ -1,135 +0,0 @@ -/* - ofxFaceTracker provides an interface to Jason Saragih's FaceTracker library. - - getImagePoint()/getImageMesh() are in image space. This means that all the - points will line up with the pixel coordinates of the image you fed into - ofxFaceTracker. - - getObjectPoint()/getObjectMesh() are in 3d object space. This is a product of - the mean mesh with only the expression applied. There is no rotation or - translation applied to the object space. - - getMeanObjectPoint()/getMeanObjectMesh() are also in 3d object space. However, - there is no expression applied to the mesh. - */ - -// Ported to Cinder version by -// vinjn (vinjn.z@gmail.com) - -#pragma once - -#include "cinder/TriMesh.h" -#include "cinder/PolyLine.h" -#include "../3rdparty/FaceTracker/Tracker.h" -#include "../IFaceTracker.h" - -using std::vector; - -namespace cinder { - typedef PolyLineT PolyLine3; -} - -#ifdef _MSC_VER -# pragma warning(disable:4244) //conversion from 'const double' to 'float' -# pragma warning(disable:4018) //signed/unsigned mismatch -#endif - -class ciFaceTracker : public ft::IFaceTracker { -public: - ciFaceTracker(); - void setup() override; - bool update(const ci::Surface& surface) override; - - void draw() const; - void reset() override; - - int size() const override; - bool getFound() const override; - bool getVisibility(int i) const; - - vector getImagePoints() const; - vector getObjectPoints() const; - vector getMeanObjectPoints() const; - - //(x,y) ~ [0,width) X [0,height) - ci::vec3 getImagePoint(int i) const override; - ci::vec3 getObjectPoint(int i) const; - ci::vec3 getMeanObjectPoint(int i) const; - - ci::TriMesh getImageMesh() const override; - ci::TriMesh getObjectMesh() const; - ci::TriMesh getMeanObjectMesh() const; - - const cv::Mat& getObjectPointsMat() const; - const ci::vec2 getImageSize() const override; - - void addTriangleIndices(ci::TriMesh& mesh) const override; - - ci::vec2 getPosition() const; - float getScale() const; - ci::vec3 getOrientation() const; - ci::mat4 getRotationMatrix() const; - - enum Direction { - FACING_FORWARD, - FACING_LEFT, FACING_RIGHT, - FACING_UNKNOWN - }; - Direction getDirection() const; - - enum Feature { - LEFT_EYEBROW, RIGHT_EYEBROW, - LEFT_EYE, RIGHT_EYE, - LEFT_JAW, RIGHT_JAW, JAW, - OUTER_MOUTH, INNER_MOUTH - }; - ci::PolyLine3 getImageFeature(Feature feature) const; - ci::PolyLine3 getObjectFeature(Feature feature) const; - ci::PolyLine3 getMeanObjectFeature(Feature feature) const; - - enum Gesture { - MOUTH_WIDTH, MOUTH_HEIGHT, - LEFT_EYEBROW_HEIGHT, RIGHT_EYEBROW_HEIGHT, - LEFT_EYE_OPENNESS, RIGHT_EYE_OPENNESS, - JAW_OPENNESS, - NOSTRIL_FLARE - }; - float getGesture(Gesture gesture) const; - - void setRescale(float rescale); - void setIterations(int iterations); - void setClamp(float clamp); - void setTolerance(float tolerance); - void setAttempts(int attempts); - void setUseInvisible(bool useInvisible); - -protected: - ci::TriMesh getMesh(const vector& points) const; - ci::TriMesh getMesh(const vector& points) const; - - void updateObjectPoints(); - - static vector getFeatureIndices(Feature feature); - ci::PolyLine3 getFeature(Feature feature, const vector& points) const; - - bool mIsFailed; - int mCurrentView; - - bool mFailCheck; - float mRescale; - int mFrameSkip; - - vector wSize1, wSize2; - int mIterations; - int mAttempts; - double mClamp, mTolerance; - bool mUseInvisible; - - FACETRACKER::Tracker mTracker; - cv::Mat mTri, mCon; - - cv::Mat mIm, mGray; - cv::Mat mObjectPoints; - ci::vec2 mImgSize; -}; - diff --git a/src/JasonFaceTracker.cpp b/src/JasonFaceTracker.cpp new file mode 100644 index 0000000..ffc528a --- /dev/null +++ b/src/JasonFaceTracker.cpp @@ -0,0 +1,365 @@ +#include "JasonFaceTracker.h" +#include "cinder/app/App.h" +#include "cinder/Utilities.h" +#include "cinder/gl/gl.h" +#include "cinder/gl/gl.h" +#include "CinderOpenCV.h" + +#include + +namespace jing +{ + using namespace ci; + using namespace std; + using namespace cv; + using namespace FACETRACKER; + +#define it at +#define db at + + // can be compiled with OpenMP for even faster threaded execution + + vector consecutive(int start, int end) { + int n = end - start; + vector result(n); + for (int i = 0; i < n; i++) { + result[i] = start + i; + } + return result; + } + + PolyLine3 ciFaceTracker::getFeature(Feature feature, const vector& points) const { + PolyLine3 polyline; + if (!mIsFailed) { + vector indices = getFeatureIndices(feature); + for (int i = 0; i < indices.size(); i++) { + int cur = indices[i]; + if (mUseInvisible || getVisibility(cur)) { + polyline.push_back(points[cur]); + } + } + } + return polyline; + } + + vector ciFaceTracker::getFeatureIndices(Feature feature) { + static int innerMouth[] = { 48, 60, 61, 62, 54, 63, 64, 65 }; + switch (feature) { + case LEFT_JAW: return consecutive(0, 9); + case RIGHT_JAW: return consecutive(8, 17); + case JAW: return consecutive(0, 17); + case LEFT_EYEBROW: return consecutive(17, 22); + case RIGHT_EYEBROW: return consecutive(22, 27); + case LEFT_EYE: return consecutive(36, 42); + case RIGHT_EYE: return consecutive(42, 48); + case OUTER_MOUTH: return consecutive(48, 60); + case INNER_MOUTH: + return vector(innerMouth, innerMouth + 8); + default: + return consecutive(0, 0); + } + } + + ciFaceTracker::ciFaceTracker() + :mRescale(1) + , mIterations(10) // [1-25] 1 is fast and inaccurate, 25 is slow and accurate + , mClamp(3) // [0-4] 1 gives a very loose fit, 4 gives a very tight fit + , mTolerance(.01) // [.01-1] match tolerance + , mAttempts(1) // [1-4] 1 is fast and may not find faces, 4 is slow but will find faces + , mIsFailed(true) + , mFailCheck(true) // check for whether the tracking failed + , mFrameSkip(-1) // how often to skip frames + , mUseInvisible(true) + { + } + + void ciFaceTracker::setup() { + wSize1.resize(1); + wSize1[0] = 7; + + wSize2.resize(3); + wSize2[0] = 11; + wSize2[1] = 9; + wSize2[2] = 7; + + string ftFile = app::getAssetPath("face2.tracker").string(); + string triFile = app::getAssetPath("face.tri").string(); + string conFile = app::getAssetPath("face.con").string(); + + mTracker.Load(ftFile.c_str()); + mTri = IO::LoadTri(triFile.c_str()); + mCon = IO::LoadCon(conFile.c_str()); // not being used right now + } + bool ciFaceTracker::update(const Surface& surface) { + Mat image = toOcv(surface); + + if (mRescale == 1) { + mIm = image; + } + else { + resize(image, mIm, cv::Size(mRescale * image.cols, mRescale * image.rows)); + } + + int nChannels = mIm.channels(); + if (nChannels == 3) + cvtColor(mIm, mGray, CV_RGB2GRAY); + else if (nChannels == 4) + cvtColor(mIm, mGray, CV_RGBA2GRAY); + else + mGray = mIm; + mImgSize = { image.cols, image.rows }; + + bool tryAgain = true; + for (int i = 0; tryAgain && i < mAttempts; i++) { + vector wSize; + if (mIsFailed) { + wSize = wSize2; + } + else { + wSize = wSize1; + } + + if (mTracker.Track(mGray, wSize, mFrameSkip, mIterations, mClamp, mTolerance, mFailCheck) == 0) { + mCurrentView = mTracker._clm.GetViewIdx(); + mIsFailed = false; + tryAgain = false; + updateObjectPoints(); + } + else { + mTracker.FrameReset(); + mIsFailed = true; + } + } + return !mIsFailed; + } + + void ciFaceTracker::draw() const { + if (mIsFailed) { + return; + } + + gl::draw(getImageMesh()); + + int n = size(); + for (int i = 0; i < n; i++) { + if (getVisibility(i)) { + gl::drawString(toString(i), { getImagePoint(i).x, getImagePoint(i).y }); + } + } + } + + void ciFaceTracker::reset() { + mTracker.FrameReset(); + } + + int ciFaceTracker::size() const { + return mTracker._shape.rows / 2; + } + + bool ciFaceTracker::getFound() const { + return !mIsFailed; + } + + bool ciFaceTracker::getVisibility(int i) const { + if (mIsFailed) { + return false; + } + const Mat& visi = mTracker._clm._visi[mCurrentView]; + return (visi.it(i, 0) != 0); + } + + vector ciFaceTracker::getImagePoints() const { + int n = size(); + vector imagePoints(n); + for (int i = 0; i < n; i++) { + imagePoints[i] = getImagePoint(i); + } + return imagePoints; + } + + vector ciFaceTracker::getObjectPoints() const { + int n = size(); + vector objectPoints(n); + for (int i = 0; i < n; i++) { + objectPoints[i] = getObjectPoint(i); + } + return objectPoints; + } + + vector ciFaceTracker::getMeanObjectPoints() const { + int n = size(); + vector meanObjectPoints(n); + for (int i = 0; i < n; i++) { + meanObjectPoints[i] = getMeanObjectPoint(i); + } + return meanObjectPoints; + } + + vec3 ciFaceTracker::getImagePoint(int i) const { + if (mIsFailed) { + return vec3(); + } + const Mat& shape = mTracker._shape; + int n = shape.rows / 2; + return vec3(shape.db(i, 0), shape.db(i + n, 0), 0.0f) / mRescale; + } + + vec3 ciFaceTracker::getObjectPoint(int i) const { + if (mIsFailed) { + return vec3(); + } + int n = mObjectPoints.rows / 3; + return vec3(mObjectPoints.db(i, 0), mObjectPoints.db(i + n, 0), mObjectPoints.db(i + n + n, 0)); + } + + vec3 ciFaceTracker::getMeanObjectPoint(int i) const { + const Mat& mean = mTracker._clm._pdm._M; + int n = mean.rows / 3; + return vec3(mean.db(i, 0), mean.db(i + n, 0), mean.db(i + n + n, 0)); + } + + TriMesh ciFaceTracker::getImageMesh() const { + return getMesh(getImagePoints()); + } + + TriMesh ciFaceTracker::getObjectMesh() const { + return getMesh(getObjectPoints()); + } + + TriMesh ciFaceTracker::getMeanObjectMesh() const { + return getMesh(getMeanObjectPoints()); + } + + const Mat& ciFaceTracker::getObjectPointsMat() const { + return mObjectPoints; + } + + vec2 ciFaceTracker::getPosition() const { + const Mat& pose = mTracker._clm._pglobl; + return vec2(pose.db(4, 0), pose.db(5, 0)) / mRescale; + } + + // multiply by ~20-23 to get pixel units (+/-20 units in the x axis, +23/-18 on the y axis) + float ciFaceTracker::getScale() const { + const Mat& pose = mTracker._clm._pglobl; + return pose.db(0, 0) / mRescale; + } + + vec3 ciFaceTracker::getOrientation() const { + const Mat& pose = mTracker._clm._pglobl; + vec3 euler(pose.db(1, 0), pose.db(2, 0), pose.db(3, 0)); + return euler; + } + + mat4 ciFaceTracker::getRotationMatrix() const { + vec3 euler = getOrientation(); + mat4 matrix = glm::eulerAngleYXZ(euler.y, euler.x, euler.z); + return matrix; + } + + ciFaceTracker::Direction ciFaceTracker::getDirection() const { + if (mIsFailed) { + return FACING_UNKNOWN; + } + switch (mCurrentView) { + case 0: return FACING_FORWARD; + case 1: return FACING_LEFT; + case 2: return FACING_RIGHT; + default: return FACING_UNKNOWN;; + } + } + + PolyLine3 ciFaceTracker::getImageFeature(Feature feature) const { + return getFeature(feature, getImagePoints()); + } + + PolyLine3 ciFaceTracker::getObjectFeature(Feature feature) const { + return getFeature(feature, getObjectPoints()); + } + + float ciFaceTracker::getGesture(Gesture gesture) const { + if (mIsFailed) { + return 0; + } + int start = 0, end = 0; + switch (gesture) { + // left to right of mouth + case MOUTH_WIDTH: start = 48; end = 54; break; + // top to bottom of inner mouth + case MOUTH_HEIGHT: start = 61; end = 64; break; + // center of the eye to middle of eyebrow + case LEFT_EYEBROW_HEIGHT: start = 38; end = 20; break; + // center of the eye to middle of eyebrow + case RIGHT_EYEBROW_HEIGHT: start = 43; end = 23; break; + // upper inner eye to lower outer eye + case LEFT_EYE_OPENNESS: start = 38; end = 41; break; + // upper inner eye to lower outer eye + case RIGHT_EYE_OPENNESS: start = 43; end = 46; break; + // nose center to chin center + case JAW_OPENNESS: start = 33; end = 8; break; + // left side of nose to right side of nose + case NOSTRIL_FLARE: start = 31; end = 35; break; + } + + return glm::length(getObjectPoint(start) - getObjectPoint(end)); + } + + void ciFaceTracker::setRescale(float rescale) { + mRescale = rescale; + } + + void ciFaceTracker::setIterations(int iterations) { + mIterations = iterations; + } + + void ciFaceTracker::setClamp(float clamp) { + mClamp = clamp; + } + + void ciFaceTracker::setTolerance(float tolerance) { + mTolerance = tolerance; + } + + void ciFaceTracker::setAttempts(int attempts) { + mAttempts = attempts; + } + + void ciFaceTracker::setUseInvisible(bool useInvisible) { + mUseInvisible = useInvisible; + } + + void ciFaceTracker::updateObjectPoints() { + const Mat& mean = mTracker._clm._pdm._M; + const Mat& variation = mTracker._clm._pdm._V; + const Mat& weights = mTracker._clm._plocal; + mObjectPoints = mean + variation * weights; + } + + void ciFaceTracker::addTriangleIndices(TriMesh& mesh) const { + for (int i = 0; i < mTri.rows; i++) { + int i0 = mTri.it(i, 0), i1 = mTri.it(i, 1), i2 = mTri.it(i, 2); + bool visible = getVisibility(i0) && getVisibility(i1) && getVisibility(i2); + if (mUseInvisible || visible) { + mesh.appendTriangle(i0, i1, i2); + } + } + } + + TriMesh ciFaceTracker::getMesh(const vector& points) const { + TriMesh mesh; + if (!mIsFailed) { + int n = size(); + for (int i = 0; i < n; i++) { + mesh.appendPosition(points[i]); + mesh.appendTexCoord({ getImagePoint(i).x / mImgSize.x, getImagePoint(i).y / mImgSize.y }); + } + addTriangleIndices(mesh); + } + return mesh; + } + + const vec2 ciFaceTracker::getImageSize() const + { + return mImgSize; + } +} \ No newline at end of file diff --git a/src/JasonFaceTracker.h b/src/JasonFaceTracker.h new file mode 100644 index 0000000..6ba89e2 --- /dev/null +++ b/src/JasonFaceTracker.h @@ -0,0 +1,131 @@ +/* + ofxFaceTracker provides an interface to Jason Saragih's FaceTracker library. + + getImagePoint()/getImageMesh() are in image space. This means that all the + points will line up with the pixel coordinates of the image you fed into + ofxFaceTracker. + + getObjectPoint()/getObjectMesh() are in 3d object space. This is a product of + the mean mesh with only the expression applied. There is no rotation or + translation applied to the object space. + + getMeanObjectPoint()/getMeanObjectMesh() are also in 3d object space. However, + there is no expression applied to the mesh. + */ + +// Ported to Cinder version by +// vinjn (vinjn.z@gmail.com) + +#pragma once + +#include "cinder/TriMesh.h" +#include "cinder/PolyLine.h" +#include "../3rdparty/FaceTracker/Tracker.h" +#include "BaseFaceTracker.h" + +namespace cinder { + typedef PolyLineT PolyLine3; +} + +namespace jing +{ + class ciFaceTracker : public BaseFaceTracker { + public: + ciFaceTracker(); + void setup() override; + bool update(const ci::Surface& surface) override; + + void draw() const; + void reset() override; + + int size() const override; + bool getFound() const override; + bool getVisibility(int i) const; + + std::vector getImagePoints() const; + std::vector getObjectPoints() const; + std::vector getMeanObjectPoints() const; + + //(x,y) ~ [0,width) X [0,height) + ci::vec3 getImagePoint(int i) const override; + ci::vec3 getObjectPoint(int i) const; + ci::vec3 getMeanObjectPoint(int i) const; + + ci::TriMesh getImageMesh() const override; + ci::TriMesh getObjectMesh() const; + ci::TriMesh getMeanObjectMesh() const; + + const cv::Mat& getObjectPointsMat() const; + const ci::vec2 getImageSize() const override; + + void addTriangleIndices(ci::TriMesh& mesh) const override; + + ci::vec2 getPosition() const; + float getScale() const; + ci::vec3 getOrientation() const; + ci::mat4 getRotationMatrix() const; + + enum Direction { + FACING_FORWARD, + FACING_LEFT, FACING_RIGHT, + FACING_UNKNOWN + }; + Direction getDirection() const; + + enum Feature { + LEFT_EYEBROW, RIGHT_EYEBROW, + LEFT_EYE, RIGHT_EYE, + LEFT_JAW, RIGHT_JAW, JAW, + OUTER_MOUTH, INNER_MOUTH + }; + ci::PolyLine3 getImageFeature(Feature feature) const; + ci::PolyLine3 getObjectFeature(Feature feature) const; + ci::PolyLine3 getMeanObjectFeature(Feature feature) const; + + enum Gesture { + MOUTH_WIDTH, MOUTH_HEIGHT, + LEFT_EYEBROW_HEIGHT, RIGHT_EYEBROW_HEIGHT, + LEFT_EYE_OPENNESS, RIGHT_EYE_OPENNESS, + JAW_OPENNESS, + NOSTRIL_FLARE + }; + float getGesture(Gesture gesture) const; + + void setRescale(float rescale); + void setIterations(int iterations); + void setClamp(float clamp); + void setTolerance(float tolerance); + void setAttempts(int attempts); + void setUseInvisible(bool useInvisible); + + protected: + ci::TriMesh getMesh(const std::vector& points) const; + ci::TriMesh getMesh(const std::vector& points) const; + + void updateObjectPoints(); + + static std::vector getFeatureIndices(Feature feature); + ci::PolyLine3 getFeature(Feature feature, const std::vector& points) const; + + bool mIsFailed; + int mCurrentView; + + bool mFailCheck; + float mRescale; + int mFrameSkip; + + std::vector wSize1, wSize2; + int mIterations; + int mAttempts; + double mClamp, mTolerance; + bool mUseInvisible; + + FACETRACKER::Tracker mTracker; + cv::Mat mTri, mCon; + + cv::Mat mIm, mGray; + cv::Mat mObjectPoints; + ci::vec2 mImgSize; + }; + +} \ No newline at end of file diff --git a/src/OpenFaceTracker.cpp b/src/OpenFaceTracker.cpp new file mode 100644 index 0000000..3c547db --- /dev/null +++ b/src/OpenFaceTracker.cpp @@ -0,0 +1,49 @@ +#include "OpenFaceTracker.h" + +namespace jing +{ + bool OpenFaceTracker::update(const ci::Surface& surface) + { + return false; + } + + void OpenFaceTracker::reset() + { + + } + + int OpenFaceTracker::size() const + { + return 0; + } + + bool OpenFaceTracker::getFound() const + { + return false; + } + + ci::vec3 OpenFaceTracker::getImagePoint(int i) const + { + return{}; + } + + ci::TriMesh OpenFaceTracker::getImageMesh() const + { + return{}; + } + + const ci::vec2 OpenFaceTracker::getImageSize() const + { + return{}; + } + + void OpenFaceTracker::addTriangleIndices(ci::TriMesh& mesh) const + { + + } + + void OpenFaceTracker::setup() + { + + } +} diff --git a/src/OpenFaceTracker.h b/src/OpenFaceTracker.h new file mode 100644 index 0000000..28eca96 --- /dev/null +++ b/src/OpenFaceTracker.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include "cinder/TriMesh.h" +#include "BaseFaceTracker.h" + +namespace jing +{ + struct OpenFaceTracker : public BaseFaceTracker + { + virtual bool update(const ci::Surface& surface); + virtual void reset(); + virtual int size() const; + virtual bool getFound() const; + virtual ci::vec3 getImagePoint(int i) const; + virtual ci::TriMesh getImageMesh() const; + virtual const ci::vec2 getImageSize() const; + virtual void addTriangleIndices(ci::TriMesh& mesh) const; + private: + virtual void setup(); + }; +} \ No newline at end of file