opencv 與dlib 結合實現人臉融合
融合流程
- 檢測面部標記
- 旋轉、縮放和轉換第二張影象，使之與第一張影象相適應
- 調整第二張影象的色彩平衡，使之與第一個相匹配
- 把第二張影象的特性混合在第一張影象中
實現流程
找到人臉矩陣
- 使用dlib提取面部標記
用Dlib實現了論文One Millisecond Face Alignment with an Ensemble of Regression Trees中的演算法
- 人臉矩陣是由68個畫素點組成，根據對應的畫素的對應相應的五官
- 1-16程式碼臉型
- 17-21 22-26 分別代表眉毛
- 27-35代表鼻子
- 36-41 42-47 程式碼眼睛
- 48-68 帶表嘴巴
get_landmarks()函式將一個影象轉化成numpy陣列，並返回一個68 x2元素矩陣，輸入影象的每個特徵點對應每行的一個x，y座標。
調整臉部影象對齊
原理
現在我們已經有了兩個標記矩陣，每行有一組座標對應一個特定的面部特徵（如第30行給出的鼻子的座標）。我們現在要搞清楚如何旋轉、翻譯和規模化第一個向量， 使它們儘可能適合第二個向量的點。想法是，可以用相同的變換在第一個影象上覆蓋第二個影象。
實現步驟
程式碼分別實現了下面幾步：
- 1．將輸入矩陣轉換為浮點數。這是之後步驟的必要條件。
- 2．每一個點集減去它的矩心。一旦為這兩個新的點集找到了一個最佳的縮放和旋轉方法，這兩個矩心c1和c2就可以用來找到完整的解決方案。
- 3．同樣，每一個點集除以它的標準偏差。這消除了問題的元件縮放偏差。
原始碼
#include <string.h>
#include <jni.h>
#include <android/log.h>
#include <string>
#include <stdio.h>
#include <vector>
#include <iostream>
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/image_processing/render_face_detections.h>
#include <dlib/image_loader/load_image.h>
#include <dlib/image_processing.h>
#include <dlib/opencv/cv_image.h>
#include <dlib/gui_widgets.h>
#include <dlib/image_io.h>
#include <dlib/opencv.h>
#include <iostream>
using namespace dlib;
using namespace std;
using namespace cv;
#define LOG_TAG "People_Det-JNI"
#define LOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, LOG_TAG, __VA_ARGS__)
#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__)
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, LOG_TAG, __VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
/*
#ifdef __cplusplus
extern "C" {
#endif
*/
static dlib::shape_predictor msp;
static frontal_face_detector detector;
static dlib::full_object_detection FACE_MARK1,FACE_MARK2;
static Mat wrap_dst, src_img2, mask;
dlib::full_object_detection detectface(dlib::cv_image<dlib::bgr_pixel> mat)
{
dlib::full_object_detection mark;
std::vector<dlib::rectangle> mRets;
mRets = detector(mat);
if(mRets.size() > 0)
mark = msp(mat, mRets[0]);
return mark;
}
jboolean facesRotate_UseData(JNIEnv* env, jobject thiz,
jintArray face1,jint w1,jint h1,
jintArray face2,jint w2,jint h2)
{
jint *cbuf1;
jint *cbuf2;
cbuf1 = env->GetIntArrayElements(face1, JNI_FALSE);
cbuf2 = env->GetIntArrayElements(face2, JNI_FALSE);
if(cbuf1 == NULL || cbuf1 == NULL){
return JNI_FALSE;
}
Mat src_img(h1, w1, CV_8UC4, (unsigned char*)cbuf1);
Mat src_img1(h2, w2, CV_8UC4, (unsigned char*)cbuf2);
cvtColor(src_img,src_img,CV_RGBA2RGB);
cvtColor(src_img1,src_img1,CV_RGBA2RGB);
if(!src_img.data || !src_img1.data)
LOGD("data error");
dlib::cv_image<dlib::bgr_pixel> img(src_img);
dlib::cv_image<dlib::bgr_pixel> img1(src_img1);
FACE_MARK1 = detectface(img);
FACE_MARK2 = detectface(img1);
std::vector<cv::Point2f> srcTri;
std::vector<cv::Point2f> dstTri;
for(int i = 0;i < 68;i++) {
srcTri.push_back(cv::Point2f(FACE_MARK1.part(i).x(),FACE_MARK1.part(i).y()));
dstTri.push_back(cv::Point2f(FACE_MARK2.part(i).x(),FACE_MARK2.part(i).y()));
}
cv::Mat warp_mat;
warp_mat = cv::findHomography(srcTri,dstTri,0);
//the faceLine's size is 68,including all face data,such as:
//the index 0 - 16:jaw data
//the index 17 - 68:face data
//the index 17 - 21:right brow data
//the index 22 - 26:left brow data
//the index 27 - 34:nose data
//the index 36 - 41:right eye data
//the index 42 - 47:left eye data
//the index 48 - 60:mouth data
src_img2 = src_img1;
cv::warpPerspective(src_img,wrap_dst,warp_mat,src_img1.size());
dlib::cv_image<dlib::bgr_pixel> img3(wrap_dst);
FACE_MARK1 = detectface(img3);
Mat new_img = Mat::zeros(src_img1.size(),CV_8UC3);
std::vector<image_window::overlay_line> faceLine;
faceLine = render_face_detections(FACE_MARK1);
for(int i = 17 ;i < 61; i++){
cv::Point p1(faceLine[i].p1.x(),faceLine[i].p1.y());
cv::Point p2(faceLine[i].p2.x(),faceLine[i].p2.y());
cv::line(new_img,p1,p2,cv::Scalar(255,255,255),1,cv::LINE_8);
}
imwrite("/data/data/com.example.faceswapdemo/files/newimg.png",new_img);
int blur_amount = faceLine[41].p1.x() - faceLine[47].p1.x();
int blur_mask = blur_amount;
if (blur_mask % 2 == 0) {
blur_mask += 1;
}
LOGD("blur_mask%d-------",blur_mask);
GaussianBlur(new_img,mask,Size(blur_mask,blur_mask),0,0);
GaussianBlur(mask,mask,Size(blur_mask,blur_mask),0,0);
//FileStorage fs("/data/data/com.example.faceswapdemo/files/face.xml",FileStorage::WRITE);
//fs << "mask" << mask;
blur_amount = blur_amount*3/5;
if (blur_amount % 2 == 0) {
blur_amount += 1;
}
LOGD("blur_amount%d-------",blur_amount);
Mat blur0 = Mat(wrap_dst.size(), wrap_dst.type());
GaussianBlur(wrap_dst, blur0, Size( blur_amount, blur_amount ), 0, 0 );
Mat blur1= Mat(src_img1.size(), src_img1.type());
GaussianBlur(src_img1, blur1, Size( blur_amount, blur_amount ), 0, 0 );
Mat out = Mat::zeros( blur1.size(), blur1.type());
for (int y = 0; y < blur1.rows; y++) {
for (int x = 0; x < blur1.cols; x++) {
for (int c = 0; c < 3; c++) {
if ((blur0.at<Vec3b>(y, x)[c]) > 0) {
out.at<Vec3b>(y, x)[c] = saturate_cast<uchar>(
(wrap_dst.at<Vec3b>(y, x)[c])
* (blur1.at<Vec3b>(y, x)[c])
/ (blur0.at<Vec3b>(y, x)[c]));
} else {
out.at<Vec3b>(y, x)[c] = saturate_cast<uchar>(
(wrap_dst.at<Vec3b>(y, x)[c])
* (blur1.at<Vec3b>(y, x)[c]));
}
}
}
}
for(int y = 0; y < src_img1.rows; y++ )
{
for(int x = 0; x < src_img1.cols; x++ )
{
if((mask.at<Vec3b>(y,x)[0]) > 0) {
for(int c = 0; c < 3; c++ )
{
src_img1.at<Vec3b>(y,x)[c]= (out.at<Vec3b>(y,x)[c]);
}
}
}
}
// src_img1 += 128 * (src_img1 <= 1.0);
//for(int i = 0; i < faceLine1.size(); i++) {
////mark the all face data
//cv::Point p1(faceLine1[i].p1.x(),faceLine[i].p1.y());
//cv::Point p2(faceLine1[i].p2.x(),faceLine[i].p2.y());
//cv::line(src_img,p1,p2,cv::Scalar(0,255,0),1,cv::LINE_8);
//}
//
//for(int i = 0; i < faceLine2.size(); i++) {
////mark the all face data
//cv::Point p1(faceLine2[i].p1.x(),faceLine[i].p1.y());
//cv::Point p2(faceLine2[i].p2.x(),faceLine[i].p2.y());
//cv::line(src_img1,p1,p2,cv::Scalar(0,255,0),1,cv::LINE_8);
//}
//Size dsize = Size(src_img1.cols,src_img1.rows);
//Mat image2 = Mat(dsize,wrap_dst.type());
//LOGD("img1 w:%d h:%d",src_img1.cols,src_img1.rows);
//resize(wrap_dst, image2,dsize);
//LOGD("dst w:%d h:%d",image2.cols,image2.rows);
imwrite("/data/data/com.example.faceswapdemo/files/data.jpg",src_img1);
env->ReleaseIntArrayElements(face1, cbuf1, 0);
env->ReleaseIntArrayElements(face2, cbuf2, 0);
return JNI_TRUE;
}
jboolean studytrain()
{
dlib::deserialize("/data/data/com.example.faceswapdemo/files/shape_predictor_68_face_landmarks.dat") >> msp;
detector = get_frontal_face_detector();
return JNI_TRUE;
}
jboolean facecheck(JNIEnv* env, jobject thiz,
jintArray face1,jint w1,jint h1,
jintArray face2,jint w2,jint h2)
{
return facesRotate_UseData(env,thiz,face1,w1,h1,face2,w2,h2);
}
jboolean removefaceswap()
{
return JNI_TRUE;
}
void startfaceswap()
{
}
void endfaceswap()
{
}
void facemapping()
{
}
void facefusion()
{
}
void faceswap()
{
}
void setfaceswapcallback()
{
}
/*jintArray facesRotate_UsePath(JNIEnv* env, jobject thiz,jstring path1,jstring path2)
{
LOGD("in det");
frontal_face_detector detector = get_frontal_face_detector();
char * str1;
str1=(char*)env->GetStringUTFChars(path1,NULL);
char * str2;
str2=(char*)env->GetStringUTFChars(path2,NULL);
LOGD("load picture");
Mat src_img = imread(str1,1);
Mat src_img1 = imread(str2,1);
LOGD("%d-----img1-----%d",src_img.cols,src_img.rows);
LOGD("%d-----img1-----%d",src_img1.cols,src_img1.rows);
LOGD("opencv img to dlib img");
if(!src_img.data || !src_img1.data)
LOGD("data error");
dlib::cv_image<dlib::bgr_pixel> img(src_img);
dlib::cv_image<dlib::bgr_pixel> img1(src_img1);
LOGD("start detector");
std::vector<dlib::rectangle> mRets;
std::vector<dlib::rectangle> mRets1;
double t = (double)getTickCount();
mRets = detector(img);
mRets1 = detector(img1);
t = ((double)getTickCount()-t)/getTickFrequency();
dlib::shape_predictor msp;
LOGD("detect use %lf s",t);
t = (double)getTickCount();
dlib::deserialize("/data/data/com.example.faceswapdemo/files/shape_predictor_68_face_landmarks.dat") >> msp;
t = ((double)getTickCount()-t)/getTickFrequency();
LOGD("load local mark use %lf s",t);
LOGD("get face mark from the picture");
t = (double)getTickCount();
dlib::full_object_detection shape1,shape2;
if (mRets.size() != 0) {
for (unsigned long j = 0; j < mRets.size(); ++j)
shape1 = msp(img, mRets[j]);
}
if (mRets1.size() != 0) {
for (unsigned long j = 0; j < mRets1.size(); ++j)
shape2 = msp(img1, mRets1[j]);
}
t = ((double)getTickCount()-t)/getTickFrequency();
LOGD("--get face mark use %lf s---",t);
LOGD("--use face mark to get mapping matrix---");
t = (double)getTickCount();
std::vector<cv::Point2f> srcTri;
std::vector<cv::Point2f> dstTri;
for(int i = 0;i < 16;i++) {
srcTri.push_back(cv::Point2f(shape1.part(i).x(),shape1.part(i).y()));
dstTri.push_back(cv::Point2f(shape2.part(i).x(),shape2.part(i).y()));
}
LOGD("--got special points---");
cv::Mat warp_mat;
warp_mat = cv::findHomography(srcTri,dstTri,0);
LOGD("%d---get change matrix-----%d",warp_mat.cols,warp_mat.rows);
uchar* p1;
for(int i = 0;i < 3;i ++) {
p1 = warp_mat.ptr<uchar>(i);
for(int j = 0;j < 3;j ++)
LOGD("matrix-----%d",(int)p1[j]);
}
//the faceLine's size is 68,including all face data,such as:
//the index 0 - 16:jaw data
//the index 17 - 68:face data
//the index 17 - 21:right brow data
//the index 22 - 26:left brow data
//the index 27 - 34:nose data
//the index 36 - 41:right eye data
//the index 42 - 47:left eye data
//the index 48 - 60:mouth data
LOGD("use the mapping matrix to change the picture");
Mat wrap_dst;
cv::warpPerspective(src_img,wrap_dst,warp_mat,src_img.size());
t = ((double)getTickCount()-t)/getTickFrequency();
LOGD("change picture use %lf s",t);
int size = wrap_dst.cols * wrap_dst.rows;
jintArray result = env->NewIntArray(size);
uchar* p;
p = wrap_dst.ptr(0);
cv::imwrite("/data/data/com.example.faceswapdemo/files/face_wrap1.png", wrap_dst);
env->SetIntArrayRegion(result, 0, size, (int*)p);
env->ReleaseStringUTFChars(path1, str1);
env->ReleaseStringUTFChars(path2, str2);
return result;
}*/
void testmat(jlong addr1,jlong addr2)
{
Mat& src = *(Mat*)addr1;
Mat& src1 = *(Mat*)addr2;
//Mat src1 = Mat(*((Mat*)addr2));
cvtColor(src1,src1,CV_RGBA2RGB);
cvtColor(src,src,CV_BGR2GRAY);
}
static JNINativeMethod gMethods[] = {
{"facecheck", "([III[III)Z", (void*)facecheck},
{"studytrain","()Z",(void*) studytrain},
// {"faceRotateUsePath", "(Ljava/lang/String;Ljava/lang/String;)[I", (void*)facesRotate_UsePath},
// {"testMat","(JJ)V",(void*) testmat},
//{"faceRotate", "()I", (void*)facesRotate},
};
/*
* 為某一個類註冊本地方法
*/
static int registerNativeMethods(JNIEnv* env
, const char* className
, JNINativeMethod* gMethods, int numMethods) {
jclass clazz;
clazz = env->FindClass(className);
if (clazz == NULL) {
return JNI_FALSE;
}
if (env->RegisterNatives(clazz, gMethods, numMethods) < 0) {
return JNI_FALSE;
}
return JNI_TRUE;
}
/*
* 為所有類註冊本地方法
*/
static int registerNatives(JNIEnv* env) {
const char* kClassName = "com/example/faceswapdemo/FaceUtils";//指定要註冊的類
return registerNativeMethods(env, kClassName, gMethods,
sizeof(gMethods)/sizeof(gMethods[0]));
}
jint JNIEXPORT JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
{
LOGE("JNI On Load");
JNIEnv* env = NULL;
jint result = -1;
if (vm->GetEnv((void**)&env, JNI_VERSION_1_4) != JNI_OK) {
LOGE("GetEnv failed!");
return result;
}
LOGE("start register!");
if (!registerNatives(env)) {//註冊
LOGE("register failed!");
return result;
}
return JNI_VERSION_1_4;
}
/*
#ifdef __cplusplus
}
#endif
*/