【OpenCV】SURF演算法之視訊影象實時特徵點匹配
阿新 • • 發佈：2019-01-08
OpenCV原始碼中有關於SURF演算法的靜態影象特徵點匹配，就將其改進為動態視訊影象實時獲取特徵點並將其與目標影象進行特徵點匹配。
考慮到如果沒有獲取到連續幀影象，即有黑色影象被攝像頭捕捉到，此時FLANN演算法則失效，因為FLANN演算法是無法處理黑色影象的，它必須能採集到特徵點時才可用，否則，程式會閃退。故在此新增如下程式碼，用來處理上述現象，並統計丟失的幀率：
static int cnt_fail = 0;
if(imageDescriptors->total == 0)
{
cnt_fail++;
continue;
}完整程式碼如下：
#include "opencv2/objdetect/objdetect.hpp"
#include "opencv2/features2d/features2d.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/calib3d/calib3d.hpp"
#include "opencv2/nonfree/nonfree.hpp"
#include "opencv2/imgproc/imgproc_c.h"
#include "opencv2/legacy/legacy.hpp"
#include "opencv2/legacy/compat.hpp"
#include <iostream>
#include <vector>
#include <stdio.h>
using namespace std;
static void help()
{
printf(
"This program demonstrated the use of the SURF Detector and Descriptor using\n"
"either FLANN (fast approx nearst neighbor classification) or brute force matching\n"
"on planar objects.\n"
"Usage:\n"
"./find_obj <object_filename> <scene_filename>, default is box.png and box_in_scene.png\n\n");
return;
}
// define whether to use approximate nearest-neighbor search
#define USE_FLANN
#ifdef USE_FLANN
static void
flannFindPairs( const CvSeq*, const CvSeq* objectDescriptors,
const CvSeq*, const CvSeq* imageDescriptors, vector<int>& ptpairs )
{
//函式flannFindPairs有5個形參1.參照物的keypoint 2.keypoint的描述符 3.圖片的keypoint 4.圖片keypoint的描述符 5.int型容器
//用於找到兩幅影象之間匹配的點對，並把匹配的點對儲存在 ptpairs 向量中，其中物體(object)影象的特徵點
int length = (int)(objectDescriptors->elem_size/sizeof(float));
cv::Mat m_object(objectDescriptors->total, length, CV_32F);
cv::Mat m_image(imageDescriptors->total, length, CV_32F);
// copy descriptors
CvSeqReader obj_reader;
float* obj_ptr = m_object.ptr<float>(0);
cvStartReadSeq( objectDescriptors, &obj_reader );//reader來讀取seq內部資料的
for(int i = 0; i < objectDescriptors->total; i++ )
{
const float* descriptor = (const float*)obj_reader.ptr;
CV_NEXT_SEQ_ELEM( obj_reader.seq->elem_size, obj_reader );
memcpy(obj_ptr, descriptor, length*sizeof(float));//memcpy記憶體拷貝函式
obj_ptr += length;
}
CvSeqReader img_reader;
float* img_ptr = m_image.ptr<float>(0);
cvStartReadSeq( imageDescriptors, &img_reader );//reader來讀取seq內部資料的
for(int i = 0; i < imageDescriptors->total; i++ )
{
const float* descriptor = (const float*)img_reader.ptr;
CV_NEXT_SEQ_ELEM( img_reader.seq->elem_size, img_reader );
memcpy(img_ptr, descriptor, length*sizeof(float));
img_ptr += length;
}
/*後加程式碼
float *database = NULL;
Mat cd = Mat(total,21,CV_32FC1,database)；
flann_index_cd.build(cd, cv::flann::KDTreeIndexParams(4),cvflann::FLANN_DIST_L2);
flann_index_cd.knnSearch(m_object, m_indices, m_dists, NUM, cv::flann::SearchParams(64) );
*/
// find nearest neighbors using FLANN
cv::Mat m_indices(objectDescriptors->total, 2, CV_32S);
cv::Mat m_dists(objectDescriptors->total, 2, CV_32F);
cv::flann::Index flann_index(m_image, cv::flann::KDTreeIndexParams(4)); // using 4 randomized kdtrees
flann_index.knnSearch(m_object, m_indices, m_dists, 2, cv::flann::SearchParams(64) ); // maximum number of leafs checked
int* indices_ptr = m_indices.ptr<int>(0);
float* dists_ptr = m_dists.ptr<float>(0);
for (int i=0;i<m_indices.rows;++i) {
if (dists_ptr[2*i]<0.6*dists_ptr[2*i+1]) {
ptpairs.push_back(i);
ptpairs.push_back(indices_ptr[2*i]);
}
}
}
#else
static double
compareSURFDescriptors( const float* d1, const float* d2, double best, int length )
{
double total_cost = 0;
assert( length % 4 == 0 );
for( int i = 0; i < length; i += 4 )
{
double t0 = d1[i ] - d2[i ];
double t1 = d1[i+1] - d2[i+1];
double t2 = d1[i+2] - d2[i+2];
double t3 = d1[i+3] - d2[i+3];
total_cost += t0*t0 + t1*t1 + t2*t2 + t3*t3;
if( total_cost > best )
break;
}
return total_cost;
}
static int
naiveNearestNeighbor( const float* vec, int laplacian,
const CvSeq* model_keypoints,
const CvSeq* model_descriptors )
{
int length = (int)(model_descriptors->elem_size/sizeof(float));
int i, neighbor = -1;
double d, dist1 = 1e6, dist2 = 1e6;
CvSeqReader reader, kreader;
cvStartReadSeq( model_keypoints, &kreader, 0 );
cvStartReadSeq( model_descriptors, &reader, 0 );
for( i = 0; i < model_descriptors->total; i++ )
{
const CvSURFPoint* kp = (const CvSURFPoint*)kreader.ptr;
const float* mvec = (const float*)reader.ptr;
CV_NEXT_SEQ_ELEM( kreader.seq->elem_size, kreader );
CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader );
if( laplacian != kp->laplacian )
continue;
d = compareSURFDescriptors( vec, mvec, dist2, length );
if( d < dist1 )
{
dist2 = dist1;
dist1 = d;
neighbor = i;
}
else if ( d < dist2 )
dist2 = d;
}
if ( dist1 < 0.6*dist2 )
return neighbor;
return -1;
}
static void
findPairs( const CvSeq* objectKeypoints, const CvSeq* objectDescriptors,
const CvSeq* imageKeypoints, const CvSeq* imageDescriptors, vector<int>& ptpairs )
{
int i;
CvSeqReader reader, kreader;
cvStartReadSeq( objectKeypoints, &kreader );
cvStartReadSeq( objectDescriptors, &reader );
ptpairs.clear();
for( i = 0; i < objectDescriptors->total; i++ )
{
const CvSURFPoint* kp = (const CvSURFPoint*)kreader.ptr;
const float* descriptor = (const float*)reader.ptr;
CV_NEXT_SEQ_ELEM( kreader.seq->elem_size, kreader );
CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader );
int nearest_neighbor = naiveNearestNeighbor( descriptor, kp->laplacian, imageKeypoints, imageDescriptors );
if( nearest_neighbor >= 0 )
{
ptpairs.push_back(i);
ptpairs.push_back(nearest_neighbor);
}
}
}
#endif
/* a rough implementation for object location */
static int
locatePlanarObject( const CvSeq* objectKeypoints, const CvSeq* objectDescriptors,
const CvSeq* imageKeypoints, const CvSeq* imageDescriptors,
const CvPoint src_corners[4], CvPoint dst_corners[4] )
// 函式locatePlanarObject有6個形參
// 1.參照物的keypoint 2.keypoint的描述符 3.圖片的keypoint 4.圖片keypoint的描述符 5.src點 6.dst點
{
double h[9];
CvMat _h = cvMat(3, 3, CV_64F, h);
vector<int> ptpairs;
vector<CvPoint2D32f> pt1, pt2;
CvMat _pt1, _pt2;
int i, n;
#ifdef USE_FLANN // 判斷是否使用Flann
flannFindPairs( objectKeypoints, objectDescriptors, imageKeypoints, imageDescriptors, ptpairs );
#else
findPairs( objectKeypoints, objectDescriptors, imageKeypoints, imageDescriptors, ptpairs );
#endif
n = (int)(ptpairs.size()/2);
if( n < 4 )
return 0;
pt1.resize(n);
pt2.resize(n);
for( i = 0; i < n; i++ )
{
pt1[i] = ((CvSURFPoint*)cvGetSeqElem(objectKeypoints,ptpairs[i*2]))->pt;
pt2[i] = ((CvSURFPoint*)cvGetSeqElem(imageKeypoints,ptpairs[i*2+1]))->pt;
}
_pt1 = cvMat(1, n, CV_32FC2, &pt1[0] );
_pt2 = cvMat(1, n, CV_32FC2, &pt2[0] );
if( !cvFindHomography( &_pt1, &_pt2, &_h, CV_RANSAC, 5 ))
//在兩個平面之間尋找單對映變換矩陣
return 0;
for( i = 0; i < 4; i++ )
{
double x = src_corners[i].x, y = src_corners[i].y;
double Z = 1./(h[6]*x + h[7]*y + h[8]);
double X = (h[0]*x + h[1]*y + h[2])*Z;
double Y = (h[3]*x + h[4]*y + h[5])*Z;
dst_corners[i] = cvPoint(cvRound(X), cvRound(Y));// cvRound浮點數取整 四捨五入
}
return 1;
}
int main(int argc, char** argv)
{
const char* object_filename = argc == 3 ? argv[1] : "E:\\Image0\\1.jpg";
const char* scene_filename = argc == 3 ? argv[2] : "scene.png";
cv::initModule_nonfree();
help();
//獲取攝像頭
CvCapture* pCapture = cvCreateCameraCapture(0);
while(1){
cvSaveImage(scene_filename,cvQueryFrame( pCapture )); // 儲存攝像機的一幀 路徑為當前資料夾下scene.png
IplImage* object = cvLoadImage( object_filename, CV_LOAD_IMAGE_GRAYSCALE ); // 載入影象
IplImage* image = cvLoadImage( scene_filename, CV_LOAD_IMAGE_GRAYSCALE ); // 載入攝像頭中儲存下來的一幀影象
if( !object || !image ) // 如果載入失敗 輸出並返回
{
fprintf( stderr, "Can not load %s or %s\n",
object_filename, scene_filename );
exit(-1);
}
CvMemStorage* storage = cvCreateMemStorage(0); // 分配新的記憶體塊
//cvNamedWindow("Object", 1);
cvNamedWindow("Object Correspond", 1);
static CvScalar colors[] =// 建立類似調色盤的東西，colors[0],表示紅色，colors[8],表示白色
{
{{0,0,255}},
{{0,128,255}},
{{0,255,255}},
{{0,255,0}},
{{255,128,0}},
{{255,255,0}},
{{255,0,0}},
{{255,0,255}},
{{255,255,255}}
};
IplImage* object_color = cvCreateImage(cvGetSize(object), 8, 3);//cvGetSize得到二維陣列的尺寸，以CvSize返回
cvCvtColor( object, object_color, CV_GRAY2BGR );//cvCvtColor 顏色空間轉換函式 將object中的圖片轉化成灰度圖並儲存在object_color中
CvSeq* objectKeypoints = 0, *objectDescriptors = 0;// 表示指向特徵點及其描述符的結構體的指標
CvSeq* imageKeypoints = 0, *imageDescriptors = 0;// CvSeq 為可動態增長元素序列，是所有OpenCV動態資料結構的基礎
int i;
CvSURFParams params = cvSURFParams(500, 1);//SURF引數設定：閾值500，生成128維描述符 SURF是尋找特徵點的演算法 內建函式
double tt = (double)cvGetTickCount();//計時
cvExtractSURF( object, 0, &objectKeypoints, &objectDescriptors, storage, params );
//提取影象中的特徵點，函式原型：
//呼叫cvExtractSURF函式
//引數1：輸入灰度圖
//引數2：mask 標誌位，指定我們識別特徵點的區域
//引數3：keypoints 向量的關鍵點
//引數4：描述符（對特徵點的屬性進行描述）
//引數5：儲存空間
//引數6：上面定義的引數
printf("Object Descriptors: %d\n", objectDescriptors->total);
//輸出Object圖片中的描述符 個數
cvExtractSURF( image, 0, &imageKeypoints, &imageDescriptors, storage, params );
printf("Image Descriptors: %d\n", imageDescriptors->total); //輸出Image圖片中的描述符 個數
static int cnt_fail = 0;
if(imageDescriptors->total == 0)
{
cnt_fail++;
continue;
}
tt = (double)cvGetTickCount() - tt;
printf( "Extraction time = %gms\n", tt/(cvGetTickFrequency()*1000.));
//計算程式執行時間，並輸出
CvPoint src_corners[4] = {{0,0}, {object->width,0}, {object->width, object->height}, {0, object->height}};
CvPoint dst_corners[4];
IplImage* correspond = cvCreateImage( cvSize(image->width, object->height+image->height), 8, 1 );
//create一個image寬度，高度為object+image的圖片（通道為1）
cvSetImageROI( correspond, cvRect( 0, 0, object->width, object->height ) );
//在圖片correspond內set感興趣區域
cvCopy( object, correspond );
//將object copy給該區域
cvSetImageROI( correspond, cvRect( 0, object->height, correspond->width, correspond->height ) );
cvCopy( image, correspond );
cvResetImageROI( correspond );
#ifdef USE_FLANN
printf("Using approximate nearest neighbor search\n");
#endif
if( locatePlanarObject( objectKeypoints, objectDescriptors, imageKeypoints,
imageDescriptors, src_corners, dst_corners ))
{
for( i = 0; i < 4; i++ )
{
CvPoint r1 = dst_corners[i%4];
CvPoint r2 = dst_corners[(i+1)%4];
cvLine( correspond, cvPoint(r1.x, r1.y+object->height ),
cvPoint(r2.x, r2.y+object->height ), colors[8],2 );
//在correspond image上畫出書的輪廓圖
}
}
vector<int> ptpairs;
#ifdef USE_FLANN
flannFindPairs( objectKeypoints, objectDescriptors, imageKeypoints, imageDescriptors, ptpairs );
#else
findPairs( objectKeypoints, objectDescriptors, imageKeypoints, imageDescriptors, ptpairs );
#endif
for( i = 0; i < (int)ptpairs.size(); i += 2 )
{
CvSURFPoint* r1 = (CvSURFPoint*)cvGetSeqElem( objectKeypoints, ptpairs[i] );
CvSURFPoint* r2 = (CvSURFPoint*)cvGetSeqElem( imageKeypoints, ptpairs[i+1] );
cvLine( correspond, cvPointFrom32f(r1->pt),
cvPoint(cvRound(r2->pt.x), cvRound(r2->pt.y+object->height)), colors[8] );
//在correspond上畫出匹配的關鍵點
}
cvShowImage( "Object Correspond", correspond ); //在視窗中顯示correspond
for( i = 0; i < objectKeypoints->total; i++ )
{
//object中特徵點的個數
CvSURFPoint* r = (CvSURFPoint*)cvGetSeqElem( objectKeypoints, i );
//返回objectKeypoints的索引，並將其強制轉化為CvSURFPoint型別
CvPoint center;//圓心
int radius;
center.x = cvRound(r->pt.x);//圓心的x
center.y = cvRound(r->pt.y);//圓心的y
radius = cvRound(r->size*1.2/9.*2);
cvCircle( object_color, center, radius, colors[0], 1, 8, 0 );
//在object_color上畫圓，圓心center，半徑radius
}
//cvShowImage( "Object", object_color );
cvWaitKey(33);
//將object_color輸出
cvReleaseImage(&image); // 釋放圖片記憶體
cvReleaseImage(&correspond); // 釋放圖片記憶體
cvReleaseImage(&object_color); // 釋放圖片記憶體
cvReleaseMemStorage(&storage); // 施放分配的記憶體
//cvReleaseCapture(&pCapture); //釋放攝像頭記憶體
}
cvReleaseCapture(&pCapture);//釋放攝像頭記憶體
cvDestroyWindow("Object");
cvDestroyWindow("Object Correspond");
return 0;
} 執行結果截圖如下：