opencv聯合dlib視訊人臉識別例子
阿新 • • 發佈:2018-12-09
本篇文章是在上一篇文章opencv聯合dlib人臉識別例子 的基礎上做了一個實時視訊人臉識別功能。
原理是利用opencv實時提取視訊中的視訊流,然後進入人臉檢測步驟,步驟類似上篇文章。
本篇文章中的程式是在VMware虛擬機器下執行的,比較卡,加入人臉識別環節導致視訊很不流暢。不過本文章中的程式碼依舊是一個視訊人臉識別的典型思路的例子。
人臉識別效果圖
工程專案目錄:
程式碼以及詳細解釋
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/image_processing/render_face_detections.h>
#include <dlib/image_processing.h>
#include <dlib/gui_widgets.h>
#include <dlib/image_io.h>
#include <dlib/opencv.h>
#include <dlib/dnn.h>
#include <dlib/data_io.h>
#include <dlib/clustering.h>
#include <dlib/string.h>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/opencv.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <iostream>
#include <vector>
#include <ctime>
#include <string>
#include <map>
#include <sstream>
#ifdef __cplusplus
extern "C"{
#endif
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <dirent.h>
#ifdef __cplusplus
}
#endif
//由於dlib和opencv中有相當一部分類同名,故不能同時對它們使用using namespace,否則會出現一些莫名其妙的問題
//且dlib庫和標準std庫中的類發生衝突,如map,string 類等等
using namespace std;
using namespace cv;
//using namespace dlib;
void getFiles(std::string path, std::map<std::string, std::string> &files);
void line_one_face_detections(cv::Mat img, std::vector<dlib::full_object_detection> fs);
//定義好一堆模板別名,以供後續方便使用
template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual = dlib::add_prev1<block<N,BN,1,dlib::tag1<SUBNET>>>;
template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual_down = dlib::add_prev2<dlib::avg_pool<2,2,2,2,dlib::skip1<dlib::tag2<block<N,BN,2,dlib::tag1<SUBNET>>>>>>;
template <int N, template <typename> class BN, int stride, typename SUBNET>
using block = BN<dlib::con<N,3,3,1,1,dlib::relu<BN<dlib::con<N,3,3,stride,stride,SUBNET>>>>>;
template <int N, typename SUBNET> using ares = dlib::relu<residual<block,N,dlib::affine,SUBNET>>;
template <int N, typename SUBNET> using ares_down = dlib::relu<residual_down<block,N,dlib::affine,SUBNET>>;
template <typename SUBNET> using alevel0 = ares_down<256,SUBNET>;
template <typename SUBNET> using alevel1 = ares<256,ares<256,ares_down<256,SUBNET>>>;
template <typename SUBNET> using alevel2 = ares<128,ares<128,ares_down<128,SUBNET>>>;
template <typename SUBNET> using alevel3 = ares<64,ares<64,ares<64,ares_down<64,SUBNET>>>>;
template <typename SUBNET> using alevel4 = ares<32,ares<32,ares<32,SUBNET>>>;
using anet_type = dlib::loss_metric<dlib::fc_no_bias<128,dlib::avg_pool_everything<
alevel0<
alevel1<
alevel2<
alevel3<
alevel4<
dlib::max_pool<3,3,2,2,dlib::relu<dlib::affine<dlib::con<32,7,7,2,2,
dlib::input_rgb_image_sized<150>
>>>>>>>>>>>>;
/*
識別視訊中的某一幀影象中是不是有庫裡的某個人
方法:
統計出庫資料夾中所有人的圖片的face_descriptors,然後計算出當前圖片中的人臉face_descriptors,二者之間距離小於0.6則視為同一個人
./t11 hls_faces huanlesong.mp4
*/
int main(int argc, char *argv[])
{
time_t start_t, end_t;
if(argc != 3)
{
std::cout<< "you should specified a dir and a video stream!"<<std::endl;
return 0;
}
time(&start_t);
std::map<string, string> files;
getFiles(argv[1], files);
if(files.empty())
{
std::cout<< "No pic files found in "<< argv[1] <<std::endl;
return 0;
}
//載入訓練好的級聯分類器,利用haar級聯分類器快速找出人臉區域,然後交給dlib檢測人臉部位
cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");
//cv::CascadeClassifier faceDetector("./output/cascade.xml");
if(faceDetector.empty())
{
std::cout << "face detector is empty!" <<std::endl;
return 0;
}
//載入人臉形狀探測器
dlib::shape_predictor sp;
dlib::deserialize("./shape_predictor_68_face_landmarks.dat") >> sp;
//載入負責人臉識別的DNN
anet_type net;
dlib::deserialize("dlib_face_recognition_resnet_model_v1.dat") >> net;
//人臉描述符庫, face_descriptor ---> name
map<dlib::matrix<float,0,1>, string> fdlib;
for(map<string, string>::iterator it = files.begin(); it != files.end(); it++ )
{
std::cout << "filename:" << it->second << " filepath:" <<it->first<<std::endl;
cv::Mat frame = cv::imread(it->first);
cv::Mat src;
cv::cvtColor(frame, src, CV_BGR2GRAY);
dlib::array2d<dlib::bgr_pixel> dimg;
dlib::assign_image(dimg, dlib::cv_image<uchar>(src));
//haar級聯分類器探測人臉區域,獲取一系列人臉所在區域
std::vector<cv::Rect> objects;
std::vector<dlib::rectangle> dets;
faceDetector.detectMultiScale(src, objects);
for (int i = 0; i < objects.size(); i++)
{
//cv::rectangle(frame, objects[i], CV_RGB(200,0,0));
dlib::rectangle r(objects[i].x, objects[i].y, objects[i].x + objects[i].width, objects[i].y + objects[i].height);
dets.push_back(r); //正常情況下應該只檢測到一副面容
}
if (dets.size() == 0)
continue;
std::vector<dlib::matrix<dlib::rgb_pixel>> faces;
std::vector<dlib::full_object_detection> shapes;
for(int i = 0; i < dets.size(); i++)
{
dlib::full_object_detection shape = sp(dimg, dets[i]); //獲取指定一個區域的人臉形狀
shapes.push_back(shape);
dlib::matrix<dlib::rgb_pixel> face_chip;
dlib::extract_image_chip(dimg, dlib::get_face_chip_details(shape,150,0.25), face_chip);
faces.push_back(move(face_chip));
}
if (faces.size() == 0)
{
cout << "No faces found in " << it->second<<endl;
continue;
}
std::vector<dlib::matrix<float,0,1>> face_descriptors = net(faces);
for(std::vector<dlib::matrix<float,0,1>>::iterator iter = face_descriptors.begin(); iter != face_descriptors.end(); iter++ )
{
fdlib.insert(pair<dlib::matrix<float,0,1>, string>(*iter, it->second));
}
}
time(&end_t);
std::cout << "ok, all pic in lib had been keep on. use time:"<< end_t - start_t << " s" <<std::endl;
//載入視訊
VideoCapture capture(argv[2]);
while(true)
{
//載入待檢測的圖片
cv::Mat frame;
capture >> frame;
if (frame.empty())
break;
cv::Mat src;
cv::cvtColor(frame, src, CV_BGR2GRAY);
dlib::array2d<dlib::bgr_pixel> dimg;
dlib::assign_image(dimg, dlib::cv_image<uchar>(src));
//haar級聯分類器探測人臉區域,獲取一系列人臉所在區域
std::vector<cv::Rect> objects;
std::vector<dlib::rectangle> dets;
faceDetector.detectMultiScale(src, objects);
for (int i = 0; i < objects.size(); i++)
{
cv::rectangle(frame, objects[i], CV_RGB(200,0,0));
dlib::rectangle r(objects[i].x, objects[i].y, objects[i].x + objects[i].width, objects[i].y + objects[i].height);
dets.push_back(r); //正常情況下應該只檢測到一副面容
}
if (dets.size() == 0)
{
continue;
}
std::vector<dlib::matrix<dlib::rgb_pixel>> faces;
std::vector<dlib::full_object_detection> shapes;
for(int i = 0; i < dets.size(); i++)
{
dlib::full_object_detection shape = sp(dimg, dets[i]); //獲取指定一個區域的人臉形狀
shapes.push_back(shape);
dlib::matrix<dlib::rgb_pixel> face_chip;
dlib::extract_image_chip(dimg, dlib::get_face_chip_details(shape,150,0.25), face_chip);
faces.push_back(move(face_chip));
}
if (faces.size() == 0)
{
continue;
}
line_one_face_detections(frame, shapes);
std::vector<dlib::matrix<float,0,1>> face_descriptors = net(faces);
//遍歷庫,查詢相似影象
float min_distance = 0.7;
std::string similar_name = "unknown";
for(map<dlib::matrix<float,0,1>, string>::iterator it=fdlib.begin(); it != fdlib.end(); it++ )
{
float distance = length(it->first - face_descriptors[0]);
if( distance < 0.5 ) //應該計算一個最近值
{
if( distance <= min_distance)
{
min_distance = distance;
similar_name = it->second;
}
}
}
if(min_distance < 0.5)
{
float similarity = (0.5 - min_distance) * 100 / 0.5;
stringstream strStream;
strStream << similar_name << ", " << similarity << '%' << endl;
string s = strStream.str();
cv::Point org(objects[0].x, objects[0].y);
cv::putText(frame, s, org, cv::FONT_HERSHEY_SIMPLEX, 1.0, CV_RGB(0, 200, 0));
}
cv::imshow("frame", frame);
//等待10ms,如果從鍵盤輸入的是q、Q、或者是Esc鍵,則退出
int key = cv::waitKey(5);
if (key == 'q' || key == 'Q' || key == 27)
break;
}
return 0;
}
void getFiles(string path, map<string, string> &files)
{
DIR *dir;
struct dirent *ptr;
char base[1000];
if(path[path.length()-1] != '/')
path = path + "/";
if((dir = opendir(path.c_str())) == NULL)
{
cout<<"open the dir: "<< path <<"error!" <<endl;
return;
}
while((ptr=readdir(dir)) !=NULL )
{
///current dir OR parrent dir
if(strcmp(ptr->d_name,".")==0 || strcmp(ptr->d_name,"..")==0)
continue;
else if(ptr->d_type == 8) //file
{
string fn(ptr->d_name);
string name;
name = fn.substr(0, fn.find_last_of("."));
string p = path + string(ptr->d_name);
files.insert(pair<string, string>(p, name));
}
else if(ptr->d_type == 10) ///link file
{}
else if(ptr->d_type == 4) ///dir
{}
}
closedir(dir);
return ;
}
void line_one_face_detections(cv::Mat img, std::vector<dlib::full_object_detection> fs)
{
int i, j;
for(j=0; j<fs.size(); j++)
{
cv::Point p1, p2;
for(i = 0; i<67; i++)
{
// 下巴到臉頰 0 ~ 16
//左邊眉毛 17 ~ 21
//右邊眉毛 21 ~ 26
//鼻樑 27 ~ 30
//鼻孔 31 ~ 35
//左眼 36 ~ 41
//右眼 42 ~ 47
//嘴脣外圈 48 ~ 59
//嘴脣內圈 59 ~ 67
switch(i)
{
case 16:
case 21:
case 26:
case 30:
case 35:
case 41:
case 47:
case 59:
i++;
break;
default:
break;
}
p1.x = fs[j].part(i).x();
p1.y = fs[j].part(i).y();
p2.x = fs[j].part(i+1).x();
p2.y = fs[j].part(i+1).y();
cv::line(img, p1, p2, cv::Scalar(0,0,255), 1);
}
}
}