diff --git a/src/FaceOffApp.cpp b/src/FaceOffApp.cpp index 38be0f1..2bc1402 100644 --- a/src/FaceOffApp.cpp +++ b/src/FaceOffApp.cpp @@ -294,9 +294,11 @@ void FaceOff::update() int nPoints = mOnlineTracker->size(); if (mFaceMesh.getBufferTexCoords0().empty()) { + auto imgSize = mOfflineTracker->getImageSize(); for (int i = 0; i < nPoints; i++) { - mFaceMesh.appendTexCoord(mOfflineTracker->getImagePoint(i) / mOfflineTracker->getImageSize()); + vec3 point = mOfflineTracker->getImagePoint(i); + mFaceMesh.appendTexCoord({ point.x / imgSize.x, point.y / imgSize.y }); } mOnlineTracker->addTriangleIndices(mFaceMesh); } @@ -304,7 +306,7 @@ void FaceOff::update() mFaceMesh.getBufferPositions().clear(); for (int i = 0; i < nPoints; i++) { - mFaceMesh.appendPosition(vec3(mOnlineTracker->getImagePoint(i), 0)); + mFaceMesh.appendPosition(mOnlineTracker->getImagePoint(i)); } if (FACE_VFX_VISIBLE && mRefTex) @@ -407,7 +409,7 @@ void FaceOff::draw() gl::ScopedModelMatrix modelMatrix; gl::ScopedColor color(ColorA(0, 1.0f, 0, 1.0f)); gl::translate(APP_W * 0.5f - adaptiveCamW * 0.5f, APP_H * 0.5f - adaptiveCamH * 0.5f); - gl::draw(mOnlineTracker->getImageMesh()); + gl::draw(mFaceMesh); if (REF_VISIBLE) { diff --git a/src/IFaceTracker/IFaceTracker.cpp b/src/IFaceTracker/IFaceTracker.cpp index eab7c7c..598117a 100644 --- a/src/IFaceTracker/IFaceTracker.cpp +++ b/src/IFaceTracker/IFaceTracker.cpp @@ -1,5 +1,5 @@ #include "IFaceTracker.h" -#include "jason_saragih_tracker/ciFacetracker.h" +#include "jason_saragih_tracker/ciFacetracker.h" namespace ft { diff --git a/src/IFaceTracker/IFaceTracker.h b/src/IFaceTracker/IFaceTracker.h index 98b25d0..b6d24c2 100644 --- a/src/IFaceTracker/IFaceTracker.h +++ b/src/IFaceTracker/IFaceTracker.h @@ -27,7 +27,7 @@ namespace ft virtual void reset() = 0; virtual int size() const = 0; virtual bool getFound() const = 0; - virtual ci::vec2 getImagePoint(int i) const = 0; + virtual ci::vec3 getImagePoint(int i) const = 0; virtual ci::TriMesh getImageMesh() const = 0; virtual const ci::vec2 getImageSize() const = 0; virtual void addTriangleIndices(ci::TriMesh& mesh) const = 0; diff --git a/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.cpp b/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.cpp index a0b0955..89a6d36 100644 --- a/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.cpp +++ b/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.cpp @@ -26,9 +26,8 @@ vector consecutive(int start, int end) { return result; } -template -ci::PolyLineT ciFaceTracker::getFeature(Feature feature, const vector& points) const { - ci::PolyLineT polyline; +PolyLine3 ciFaceTracker::getFeature(Feature feature, const vector& points) const { + PolyLine3 polyline; if (!mIsFailed) { vector indices = getFeatureIndices(feature); for (int i = 0; i < indices.size(); i++) { @@ -42,33 +41,33 @@ ci::PolyLineT ciFaceTracker::getFeature(Feature feature, const vector& poi } vector ciFaceTracker::getFeatureIndices(Feature feature) { - static int innerMouth[] = {48,60,61,62,54,63,64,65}; - switch(feature) { - case LEFT_JAW: return consecutive(0, 9); - case RIGHT_JAW: return consecutive(8, 17); - case JAW: return consecutive(0, 17); - case LEFT_EYEBROW: return consecutive(17, 22); - case RIGHT_EYEBROW: return consecutive(22, 27); - case LEFT_EYE: return consecutive(36, 42); - case RIGHT_EYE: return consecutive(42, 48); - case OUTER_MOUTH: return consecutive(48, 60); - case INNER_MOUTH: - return vector(innerMouth, innerMouth + 8); - default: - return consecutive(0,0); + static int innerMouth[] = { 48, 60, 61, 62, 54, 63, 64, 65 }; + switch (feature) { + case LEFT_JAW: return consecutive(0, 9); + case RIGHT_JAW: return consecutive(8, 17); + case JAW: return consecutive(0, 17); + case LEFT_EYEBROW: return consecutive(17, 22); + case RIGHT_EYEBROW: return consecutive(22, 27); + case LEFT_EYE: return consecutive(36, 42); + case RIGHT_EYE: return consecutive(42, 48); + case OUTER_MOUTH: return consecutive(48, 60); + case INNER_MOUTH: + return vector(innerMouth, innerMouth + 8); + default: + return consecutive(0, 0); } } ciFaceTracker::ciFaceTracker() -:mRescale(1) -,mIterations(10) // [1-25] 1 is fast and inaccurate, 25 is slow and accurate -,mClamp(3) // [0-4] 1 gives a very loose fit, 4 gives a very tight fit -,mTolerance(.01) // [.01-1] match tolerance -,mAttempts(1) // [1-4] 1 is fast and may not find faces, 4 is slow but will find faces -,mIsFailed(true) -,mFailCheck(true) // check for whether the tracking failed -,mFrameSkip(-1) // how often to skip frames -,mUseInvisible(true) + :mRescale(1) + , mIterations(10) // [1-25] 1 is fast and inaccurate, 25 is slow and accurate + , mClamp(3) // [0-4] 1 gives a very loose fit, 4 gives a very tight fit + , mTolerance(.01) // [.01-1] match tolerance + , mAttempts(1) // [1-4] 1 is fast and may not find faces, 4 is slow but will find faces + , mIsFailed(true) + , mFailCheck(true) // check for whether the tracking failed + , mFrameSkip(-1) // how often to skip frames + , mUseInvisible(true) { } @@ -89,12 +88,13 @@ void ciFaceTracker::setup() { mTri = IO::LoadTri(triFile.c_str()); mCon = IO::LoadCon(conFile.c_str()); // not being used right now } -bool ciFaceTracker::update(const ci::Surface& surface) { +bool ciFaceTracker::update(const Surface& surface) { Mat image = toOcv(surface); if (mRescale == 1) { - mIm = image; - } else { + mIm = image; + } + else { resize(image, mIm, cv::Size(mRescale * image.cols, mRescale * image.rows)); } @@ -112,7 +112,8 @@ bool ciFaceTracker::update(const ci::Surface& surface) { vector wSize; if (mIsFailed) { wSize = wSize2; - } else { + } + else { wSize = wSize1; } @@ -121,7 +122,8 @@ bool ciFaceTracker::update(const ci::Surface& surface) { mIsFailed = false; tryAgain = false; updateObjectPoints(); - } else { + } + else { mTracker.FrameReset(); mIsFailed = true; } @@ -134,12 +136,12 @@ void ciFaceTracker::draw() const{ return; } - ci::gl::draw(getImageMesh()); + gl::draw(getImageMesh()); int n = size(); for (int i = 0; i < n; i++){ - if (getVisibility(i)) { - ci::gl::drawString(ci::toString(i), getImagePoint(i)); + if (getVisibility(i)) { + gl::drawString(toString(i), { getImagePoint(i).x, getImagePoint(i).y }); } } } @@ -164,65 +166,65 @@ bool ciFaceTracker::getVisibility(int i) const { return (visi.it(i, 0) != 0); } -vector ciFaceTracker::getImagePoints() const { +vector ciFaceTracker::getImagePoints() const { int n = size(); - vector imagePoints(n); + vector imagePoints(n); for (int i = 0; i < n; i++) { imagePoints[i] = getImagePoint(i); } return imagePoints; } -vector ciFaceTracker::getObjectPoints() const { +vector ciFaceTracker::getObjectPoints() const { int n = size(); - vector objectPoints(n); + vector objectPoints(n); for (int i = 0; i < n; i++) { objectPoints[i] = getObjectPoint(i); } return objectPoints; } -vector ciFaceTracker::getMeanObjectPoints() const { +vector ciFaceTracker::getMeanObjectPoints() const { int n = size(); - vector meanObjectPoints(n); + vector meanObjectPoints(n); for (int i = 0; i < n; i++) { meanObjectPoints[i] = getMeanObjectPoint(i); } return meanObjectPoints; } -ci::vec2 ciFaceTracker::getImagePoint(int i) const { +vec3 ciFaceTracker::getImagePoint(int i) const { if (mIsFailed) { - return ci::vec2(); + return vec3(); } const Mat& shape = mTracker._shape; int n = shape.rows / 2; - return ci::vec2(shape.db(i, 0), shape.db(i + n, 0)) / mRescale; + return vec3(shape.db(i, 0), shape.db(i + n, 0), 0.0f) / mRescale; } -ci::vec3 ciFaceTracker::getObjectPoint(int i) const { +vec3 ciFaceTracker::getObjectPoint(int i) const { if (mIsFailed) { - return ci::vec3(); - } + return vec3(); + } int n = mObjectPoints.rows / 3; - return ci::vec3(mObjectPoints.db(i,0), mObjectPoints.db(i+n,0), mObjectPoints.db(i+n+n,0)); + return vec3(mObjectPoints.db(i, 0), mObjectPoints.db(i + n, 0), mObjectPoints.db(i + n + n, 0)); } -ci::vec3 ciFaceTracker::getMeanObjectPoint(int i) const { +vec3 ciFaceTracker::getMeanObjectPoint(int i) const { const Mat& mean = mTracker._clm._pdm._M; int n = mean.rows / 3; - return ci::vec3(mean.db(i,0), mean.db(i+n,0), mean.db(i+n+n,0)); + return vec3(mean.db(i, 0), mean.db(i + n, 0), mean.db(i + n + n, 0)); } -ci::TriMesh ciFaceTracker::getImageMesh() const{ +TriMesh ciFaceTracker::getImageMesh() const{ return getMesh(getImagePoints()); } -ci::TriMesh ciFaceTracker::getObjectMesh() const { +TriMesh ciFaceTracker::getObjectMesh() const { return getMesh(getObjectPoints()); } -ci::TriMesh ciFaceTracker::getMeanObjectMesh() const { +TriMesh ciFaceTracker::getMeanObjectMesh() const { return getMesh(getMeanObjectPoints()); } @@ -230,26 +232,26 @@ const Mat& ciFaceTracker::getObjectPointsMat() const { return mObjectPoints; } -ci::vec2 ciFaceTracker::getPosition() const { +vec2 ciFaceTracker::getPosition() const { const Mat& pose = mTracker._clm._pglobl; - return ci::vec2(pose.db(4,0), pose.db(5,0)) / mRescale; + return vec2(pose.db(4, 0), pose.db(5, 0)) / mRescale; } // multiply by ~20-23 to get pixel units (+/-20 units in the x axis, +23/-18 on the y axis) float ciFaceTracker::getScale() const { const Mat& pose = mTracker._clm._pglobl; - return pose.db(0,0) / mRescale; + return pose.db(0, 0) / mRescale; } -ci::vec3 ciFaceTracker::getOrientation() const { +vec3 ciFaceTracker::getOrientation() const { const Mat& pose = mTracker._clm._pglobl; - ci::vec3 euler(pose.db(1, 0), pose.db(2, 0), pose.db(3, 0)); + vec3 euler(pose.db(1, 0), pose.db(2, 0), pose.db(3, 0)); return euler; } -ci::mat4 ciFaceTracker::getRotationMatrix() const { - ci::vec3 euler = getOrientation(); - ci::mat4 matrix = glm::eulerAngleYXZ(euler.y, euler.x, euler.z); +mat4 ciFaceTracker::getRotationMatrix() const { + vec3 euler = getOrientation(); + mat4 matrix = glm::eulerAngleYXZ(euler.y, euler.x, euler.z); return matrix; } @@ -257,48 +259,44 @@ ciFaceTracker::Direction ciFaceTracker::getDirection() const { if (mIsFailed) { return FACING_UNKNOWN; } - switch(mCurrentView) { - case 0: return FACING_FORWARD; - case 1: return FACING_LEFT; - case 2: return FACING_RIGHT; - default: return FACING_UNKNOWN;; + switch (mCurrentView) { + case 0: return FACING_FORWARD; + case 1: return FACING_LEFT; + case 2: return FACING_RIGHT; + default: return FACING_UNKNOWN;; } } -ci::PolyLine2 ciFaceTracker::getImageFeature(Feature feature) const { +PolyLine3 ciFaceTracker::getImageFeature(Feature feature) const { return getFeature(feature, getImagePoints()); } -ci::PolyLine3 ciFaceTracker::getObjectFeature(Feature feature) const { +PolyLine3 ciFaceTracker::getObjectFeature(Feature feature) const { return getFeature(feature, getObjectPoints()); } -ci::PolyLine3 ciFaceTracker::getMeanObjectFeature(Feature feature) const { - return getFeature(feature, getMeanObjectPoints()); -} - float ciFaceTracker::getGesture(Gesture gesture) const { if (mIsFailed) { return 0; } int start = 0, end = 0; - switch(gesture) { + switch (gesture) { // left to right of mouth - case MOUTH_WIDTH: start = 48; end = 54; break; - // top to bottom of inner mouth - case MOUTH_HEIGHT: start = 61; end = 64; break; - // center of the eye to middle of eyebrow - case LEFT_EYEBROW_HEIGHT: start = 38; end = 20; break; - // center of the eye to middle of eyebrow - case RIGHT_EYEBROW_HEIGHT: start = 43; end = 23; break; - // upper inner eye to lower outer eye - case LEFT_EYE_OPENNESS: start = 38; end = 41; break; - // upper inner eye to lower outer eye - case RIGHT_EYE_OPENNESS: start = 43; end = 46; break; - // nose center to chin center - case JAW_OPENNESS: start = 33; end = 8; break; - // left side of nose to right side of nose - case NOSTRIL_FLARE: start = 31; end = 35; break; + case MOUTH_WIDTH: start = 48; end = 54; break; + // top to bottom of inner mouth + case MOUTH_HEIGHT: start = 61; end = 64; break; + // center of the eye to middle of eyebrow + case LEFT_EYEBROW_HEIGHT: start = 38; end = 20; break; + // center of the eye to middle of eyebrow + case RIGHT_EYEBROW_HEIGHT: start = 43; end = 23; break; + // upper inner eye to lower outer eye + case LEFT_EYE_OPENNESS: start = 38; end = 41; break; + // upper inner eye to lower outer eye + case RIGHT_EYE_OPENNESS: start = 43; end = 46; break; + // nose center to chin center + case JAW_OPENNESS: start = 33; end = 8; break; + // left side of nose to right side of nose + case NOSTRIL_FLARE: start = 31; end = 35; break; } return glm::length(getObjectPoint(start) - getObjectPoint(end)); @@ -335,7 +333,7 @@ void ciFaceTracker::updateObjectPoints() { mObjectPoints = mean + variation * weights; } -void ciFaceTracker::addTriangleIndices(ci::TriMesh& mesh) const { +void ciFaceTracker::addTriangleIndices(TriMesh& mesh) const { for (int i = 0; i < mTri.rows; i++) { int i0 = mTri.it(i, 0), i1 = mTri.it(i, 1), i2 = mTri.it(i, 2); bool visible = getVisibility(i0) && getVisibility(i1) && getVisibility(i2); @@ -345,34 +343,20 @@ void ciFaceTracker::addTriangleIndices(ci::TriMesh& mesh) const { } } - -ci::TriMesh ciFaceTracker::getMesh(const vector& points) const { - ci::TriMesh mesh; +TriMesh ciFaceTracker::getMesh(const vector& points) const { + TriMesh mesh; if (!mIsFailed) { int n = size(); for (int i = 0; i < n; i++) { mesh.appendPosition(points[i]); - mesh.appendTexCoord(getImagePoint(i)/mImgSize); - } - addTriangleIndices(mesh); - } - return mesh; -} - -ci::TriMesh ciFaceTracker::getMesh(const vector& points) const { - ci::TriMesh mesh; - if (!mIsFailed) { - int n = size(); - for (int i = 0; i < n; i++) { - mesh.appendPosition(ci::vec3(points[i],0)); - mesh.appendTexCoord(getImagePoint(i)/mImgSize); + mesh.appendTexCoord({ getImagePoint(i).x / mImgSize.x, getImagePoint(i).y / mImgSize.y }); } addTriangleIndices(mesh); } return mesh; } -const ci::vec2 ciFaceTracker::getImageSize() const +const vec2 ciFaceTracker::getImageSize() const { return mImgSize; } diff --git a/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.h b/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.h index 71b9b1d..851e31e 100644 --- a/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.h +++ b/src/IFaceTracker/jason_saragih_tracker/ciFaceTracker.h @@ -1,14 +1,14 @@ /* ofxFaceTracker provides an interface to Jason Saragih's FaceTracker library. - + getImagePoint()/getImageMesh() are in image space. This means that all the points will line up with the pixel coordinates of the image you fed into ofxFaceTracker. - + getObjectPoint()/getObjectMesh() are in 3d object space. This is a product of the mean mesh with only the expression applied. There is no rotation or translation applied to the object space. - + getMeanObjectPoint()/getMeanObjectMesh() are also in 3d object space. However, there is no expression applied to the mesh. */ @@ -32,101 +32,100 @@ using std::vector; class ciFaceTracker : public ft::IFaceTracker { public: - ciFaceTracker(); + ciFaceTracker(); void setup() override; bool update(const ci::Surface& surface) override; - void draw() const; + void draw() const; void reset() override; - + int size() const override; bool getFound() const override; - bool getVisibility(int i) const; - - vector getImagePoints() const; - vector getObjectPoints() const; - vector getMeanObjectPoints() const; - - //(x,y) ~ [0,width) X [0,height) - ci::vec2 getImagePoint(int i) const override; - ci::vec3 getObjectPoint(int i) const; - ci::vec3 getMeanObjectPoint(int i) const; - + bool getVisibility(int i) const; + + vector getImagePoints() const; + vector getObjectPoints() const; + vector getMeanObjectPoints() const; + + //(x,y) ~ [0,width) X [0,height) + ci::vec3 getImagePoint(int i) const override; + ci::vec3 getObjectPoint(int i) const; + ci::vec3 getMeanObjectPoint(int i) const; + ci::TriMesh getImageMesh() const override; - ci::TriMesh getObjectMesh() const; - ci::TriMesh getMeanObjectMesh() const; - - const cv::Mat& getObjectPointsMat() const; + ci::TriMesh getObjectMesh() const; + ci::TriMesh getMeanObjectMesh() const; + + const cv::Mat& getObjectPointsMat() const; const ci::vec2 getImageSize() const override; void addTriangleIndices(ci::TriMesh& mesh) const override; - - ci::vec2 getPosition() const; - float getScale() const; - ci::vec3 getOrientation() const; - ci::mat4 getRotationMatrix() const; - - enum Direction { - FACING_FORWARD, - FACING_LEFT, FACING_RIGHT, - FACING_UNKNOWN - }; - Direction getDirection() const; - - enum Feature { - LEFT_EYEBROW, RIGHT_EYEBROW, - LEFT_EYE, RIGHT_EYE, - LEFT_JAW, RIGHT_JAW, JAW, - OUTER_MOUTH, INNER_MOUTH - }; - ci::PolyLine2 getImageFeature(Feature feature) const; - ci::PolyLine3 getObjectFeature(Feature feature) const; - ci::PolyLine3 getMeanObjectFeature(Feature feature) const; - - enum Gesture { - MOUTH_WIDTH, MOUTH_HEIGHT, - LEFT_EYEBROW_HEIGHT, RIGHT_EYEBROW_HEIGHT, - LEFT_EYE_OPENNESS, RIGHT_EYE_OPENNESS, - JAW_OPENNESS, - NOSTRIL_FLARE - }; - float getGesture(Gesture gesture) const; - - void setRescale(float rescale); - void setIterations(int iterations); - void setClamp(float clamp); - void setTolerance(float tolerance); - void setAttempts(int attempts); - void setUseInvisible(bool useInvisible); - + + ci::vec2 getPosition() const; + float getScale() const; + ci::vec3 getOrientation() const; + ci::mat4 getRotationMatrix() const; + + enum Direction { + FACING_FORWARD, + FACING_LEFT, FACING_RIGHT, + FACING_UNKNOWN + }; + Direction getDirection() const; + + enum Feature { + LEFT_EYEBROW, RIGHT_EYEBROW, + LEFT_EYE, RIGHT_EYE, + LEFT_JAW, RIGHT_JAW, JAW, + OUTER_MOUTH, INNER_MOUTH + }; + ci::PolyLine3 getImageFeature(Feature feature) const; + ci::PolyLine3 getObjectFeature(Feature feature) const; + ci::PolyLine3 getMeanObjectFeature(Feature feature) const; + + enum Gesture { + MOUTH_WIDTH, MOUTH_HEIGHT, + LEFT_EYEBROW_HEIGHT, RIGHT_EYEBROW_HEIGHT, + LEFT_EYE_OPENNESS, RIGHT_EYE_OPENNESS, + JAW_OPENNESS, + NOSTRIL_FLARE + }; + float getGesture(Gesture gesture) const; + + void setRescale(float rescale); + void setIterations(int iterations); + void setClamp(float clamp); + void setTolerance(float tolerance); + void setAttempts(int attempts); + void setUseInvisible(bool useInvisible); + protected: ci::TriMesh getMesh(const vector& points) const; ci::TriMesh getMesh(const vector& points) const; - void updateObjectPoints(); - - static vector getFeatureIndices(Feature feature); - template - ci::PolyLineT getFeature(Feature feature, const vector& points) const; - - bool mIsFailed; - int mCurrentView; - - bool mFailCheck; - float mRescale; - int mFrameSkip; - - vector wSize1, wSize2; - int mIterations; - int mAttempts; - double mClamp, mTolerance; - bool mUseInvisible; - - FACETRACKER::Tracker mTracker; - cv::Mat mTri, mCon; - - cv::Mat mIm, mGray; - cv::Mat mObjectPoints; - ci::vec2 mImgSize; + void updateObjectPoints(); + + static vector getFeatureIndices(Feature feature); + ci::PolyLine3 getFeature(Feature feature, const vector& points) const; + + bool mIsFailed; + int mCurrentView; + + bool mFailCheck; + float mRescale; + int mFrameSkip; + + vector wSize1, wSize2; + int mIterations; + int mAttempts; + double mClamp, mTolerance; + bool mUseInvisible; + + FACETRACKER::Tracker mTracker; + cv::Mat mTri, mCon; + + cv::Mat mIm, mGray; + cv::Mat mObjectPoints; + ci::vec2 mImgSize; };