相关
ffmpeg解码示例
x264编码示例
ffmpeg编码示例
多视频叠加
人脸识别免费框架
使用《大话西游》中一帧作为素材
素材
opencv
opencv中demo较为全面，人脸检测使用对应的haarcascades实现。
dlib
检测范围更多，精度与效果更好。但是需要使用到100mb的模型数据库，移动应用上使用不太现实。上图效果同样使用face_landmark_detectionx_ex 这个官方提供的demo。
flandmark
可以实现8个关键点检测，效果和效率一般，不过检测全面，而且模型只有5mb。
实现代码：
#include <opencv/cv.h>
#include <opencv/cvaux.h>
#include <opencv/highgui.h>
#include <cstring>
#include <cmath>
#include "flandmark_detector.h"
void detectFaceInImage(IplImage orig, IplImage input, CvHaarClassifierCascade* cascade, FLANDMARK_Model model, int bbox, double landmarks)
{
// Smallest face size.
CvSize minFeatureSize = cvSize(60, 60);
int flags = CV_HAAR_DO_CANNY_PRUNING;
// How detailed should the search be.
float search_scale_factor = 1.2f;
CvMemStorage storage;
CvSeq rects;
int nFaces;
storage = cvCreateMemStorage(0);
cvClearMemStorage(storage);
double t = (double)cvGetTickCount();
// Detect all the faces in the greyscale image.
rects = cvHaarDetectObjects(input, cascade, storage, search_scale_factor, 3, 0, minFeatureSize);
nFaces = rects->total;
printf("face = %d",nFaces);
for (int iface = 0; iface < (rects ? nFaces : 0); ++iface)
{
CvRect r = (CvRect)cvGetSeqElem(rects, iface);
bbox[0] = r->x;
bbox[1] = r->y;
bbox[2] = r->x + r->width;
bbox[3] = r->y + r->height;
flandmark_detect(input, bbox, model, landmarks);
// display landmarks
cvRectangle(orig, cvPoint(bbox[0], bbox[1]), cvPoint(bbox[2], bbox[3]), CV_RGB(255,0,0) );
cvRectangle(orig, cvPoint(model->bb[0], model->bb[1]), cvPoint(model->bb[2], model->bb[3]), CV_RGB(0,0,255) );
cvCircle(orig, cvPoint((int)landmarks[0], (int)landmarks[1]), 3, CV_RGB(0, 0,255), CV_FILLED);
for (int i = 2; i < 2model->data.options.M; i += 2)
{
cvCircle(orig, cvPoint(int(landmarks[i]), int(landmarks[i+1])), 3, CV_RGB(255,0,0), CV_FILLED);
}
}
t = (double)cvGetTickCount() - t;
int ms = cvRound( t / ((double)cvGetTickFrequency() * 1000.0) );
if (nFaces > 0)
{
printf("Faces detected: %d; Detection of facial landmark on all faces took %d ms\n", nFaces, ms);
} else {
printf("NO Face\n");
}
cvReleaseMemStorage(&storage);
}
int main( int argc, char** argv )
{
char flandmark_window[] = "flandmark_example1";
double t;
int ms;
if (argc < 2)
{
fprintf(stderr, "Usage: flandmark_1 <path_to_input_image> [<path_to_output_image>]\n");
exit(1);
}
char faceCascadeFilename[] = "haarcascade_frontalface_alt.xml";
// Load the HaarCascade classifier for face detection.
CvHaarClassifierCascade* faceCascade;
faceCascade = (CvHaarClassifierCascade)cvLoad(faceCascadeFilename, 0, 0, 0);
if( !faceCascade )
{
printf("Couldnt load Face detector '%s'\n", faceCascadeFilename);
exit(1);
}
t = (double)cvGetTickCount();
FLANDMARK_Model * model = flandmark_init("flandmark_model.dat");
if (model == 0)
{
printf("Structure model wasn't created. Corrupted file flandmark_model.dat?\n");
exit(1);
}
t = (double)cvGetTickCount() - t;
ms = cvRound( t / ((double)cvGetTickFrequency() * 1000.0) );
printf("Structure model loaded in %d ms.\n", ms);
// ------------- end flandmark load model
// input image
IplImage frame = cvLoadImage(argv[1]);
if (frame == NULL)
{
fprintf(stderr, "Cannot open image %s. Exiting...\n", argv[1]);
exit(1);
}
// convert image to grayscale
IplImage frame_bw = cvCreateImage(cvSize(frame->width, frame->height), IPL_DEPTH_8U, 1);
cvConvertImage(frame, frame_bw);
int bbox = (int)malloc(4sizeof(int));
double landmarks = (double)malloc(2model->data.options.Msizeof(double));
detectFaceInImage(frame, frame_bw, faceCascade, model, bbox, landmarks);
// cvShowImage(flandmark_window, frame);
// cvWaitKey(0);
if (argc == 3)
{
printf("Saving image to file %s...\n", argv[2]);
cvSaveImage(argv[2], frame);
}
// cleanup
free(bbox);
free(landmarks);
// cvDestroyWindow(flandmark_window);
cvReleaseImage(&frame);
cvReleaseImage(&frame_bw);
cvReleaseHaarClassifierCascade(&faceCascade);
flandmark_free(model);
}
flandmark就是基于opencv二次开发的，人脸检测这个步骤就是使用的oepncv的接口实现。
另外收费的人脸关键点检测的技术提供还有很多。
动态人脸贴纸
素材视频
input
输出效果
output
代码实现
#include <opencv2/objdetect.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <stdio.h>
#include <iostream>
using namespace std;
using namespace cv;
void write_yuv_pic(Mat& img, CascadeClassifier& cascade,
CascadeClassifier& nestedCascade,
double scale, bool tryflip,FILE* out);
int is_yuv_file(const char * filename) ;
int width = 480,height = 480;
string cascadeName;
string nestedCascadeName;
int main( int argc, char** argv )
{
VideoCapture capture;
Mat frame, image;
char* inputName,outPutName;
bool tryflip;
CascadeClassifier cascade, nestedCascade;
double scale;
cascadeName = "./haarcascades/haarcascade_frontalface_alt.xml";
nestedCascadeName = "./haarcascades/haarcascade_eye_tree_eyeglasses.xml";
scale = 1;
tryflip = false;
if(argc !=3)
{
puts("useage:argv[0] yuv_input_file yuv_output_file");
return -1;
}
inputName = argv[1];
outPutName = argv[2];
int index = strlen(inputName)-1;
if(!(is_yuv_file(inputName)&&is_yuv_file(outPutName)))
{
puts("useage:argv[0] yuv_input_file yuv_output_file");
return -1;
}
int i = 0;
FILE * out = (FILE)fopen(outPutName,"wb+");
FILE * file = (FILE )fopen(inputName,"rb");
unsigned char * yuvbuff = (unsigned char )malloc(widthheight3/2);
cerr << cascadeName << endl << nestedCascadeName <<endl << inputName<<endl;
if ( !nestedCascade.load( nestedCascadeName ) )
cerr << "WARNING: Could not load classifier cascade for nested objects" << endl;
if( !cascade.load( cascadeName ) )
{
cerr << "ERROR: Could not load classifier cascade" << endl;
return -1;
}
for(i=0;i < 100;i++)
{
fread(yuvbuff,1,widthheight3/2,file);
Mat yuvMat(height+height/2,width,CV_8UC1,yuvbuff);
cvtColor(yuvMat,image,CV_YUV420p2RGB);
if( !image.empty() )
{
write_yuv_pic( image, cascade, nestedCascade, scale, tryflip,out);
}
}
fclose(file);
fclose(out);
free(yuvbuff);
return 0;
}
void write_yuv_pic(Mat& img, CascadeClassifier& cascade,
CascadeClassifier& nestedCascade,
double scale, bool tryflip,FILE* out)
{
double t = 0;
vector<Rect> faces, faces2;
const static Scalar colors[] =
{
Scalar(255,0,0),
Scalar(255,128,0),
Scalar(255,255,0),
Scalar(0,255,0),
Scalar(0,128,255),
Scalar(0,255,255),
Scalar(0,0,255),
Scalar(255,0,255)
};
Mat gray, smallImg;
cvtColor( img, gray, COLOR_BGR2GRAY );
double fx = 1 / scale;
resize( gray, smallImg, Size(), fx, fx, INTER_LINEAR );
equalizeHist( smallImg, smallImg );
t = (double)cvGetTickCount();
cascade.detectMultiScale( smallImg, faces,
1.1, 3, 0
//|CASCADE_FIND_BIGGEST_OBJECT
//|CASCADE_DO_ROUGH_SEARCH
|CASCADE_SCALE_IMAGE
,
Size(80, 80) );
t = (double)cvGetTickCount() - t;
printf( "detection time = %g ms\n", t/((double)cvGetTickFrequency()1000.) );
for ( size_t i = 0; i < faces.size(); i++ )
{
Rect r = faces[i];
Mat smallImgROI;
vector<Rect> nestedObjects;
Point center;
Scalar color = colors[i%8];
int radius;
double aspect_ratio = (double)r.width/r.height;
if( 0.75 < aspect_ratio && aspect_ratio < 1.3 )
{
Point s ;
IplImage pimg = IplImage(img);
IplImage topimage = cvLoadImage( "head_top.png", 1 );
int iwidth = (topimage).width;
int iheight = (topimage).height;
printf("s.x = %d,s.y = %d,width = %d,height = %d\n",s.x,s.y,iwidth,iheight);
s.y= r.y-iheight;
s.x = r.x-(iwidth-r.width)/2;
if(s.y < 0 )
s.y = 0;
if(s.x < 0 )
s.x = 0;
printf("s.x = %d,s.y = %d,width = %d,height = %d\n",s.x,s.y,iwidth,iheight);
cvSetImageROI(topimage,CvRect(0,0,iwidth,iheight));
cvSetImageROI(&pimg,CvRect(s.x,s.y,iwidth,iheight));
cvAddWeighted(&pimg,1,topimage,0.5,0.0,&pimg);
cvResetImageROI(&pimg);
cvResetImageROI(topimage);
}
if( nestedCascade.empty() )
continue;
smallImgROI = smallImg( r );
t = (double)cvGetTickCount();
nestedCascade.detectMultiScale( smallImgROI, nestedObjects,
1.5, 3, 0
//|CASCADE_FIND_BIGGEST_OBJECT
//|CASCADE_DO_ROUGH_SEARCH
//|CASCADE_DO_CANNY_PRUNING
|CASCADE_SCALE_IMAGE
,
Size(20, 20) );
t = (double)cvGetTickCount() - t;
printf( "nestdetection time = %g ms\n", t/((double)cvGetTickFrequency()1000.) );
for ( size_t j = 0; j < nestedObjects.size(); j++ )
{
Rect nr = nestedObjects[j];
IplImage pimg = IplImage(img);
IplImage eyeimage = cvLoadImage( "eye.png", 1 );
int iwidth = (eyeimage).width;
int iheight = (eyeimage).height;
int x = nr.x+r.x ;
int y = nr.y +nr.height+r.y;
printf("s.x = %d,s.y = %d,width = %d,height = %d\n",x,y,iwidth,iheight);
cvSetImageROI(eyeimage,CvRect(0,0,iwidth,iheight-2));
cvSetImageROI(&pimg,CvRect(x,y,iwidth,iheight-2));
cvAddWeighted(&pimg,1,eyeimage,0.5,0.0,&pimg);
cvResetImageROI(&pimg);
cvResetImageROI(eyeimage);
}
}
unsigned char * yuvbuff = (unsigned char )malloc(widthheight3/2);
Mat yuvMat(height+height/2,width,CV_8UC1,yuvbuff);
cvtColor(img,yuvMat,CV_BGR2YUV_I420);
fwrite(yuvMat.data,1,widthheight*3/2,out);
free(yuvbuff);
}
int is_yuv_file(const char * filename) {
int i = 0;
char * end = ".yuv";
if(filename == NULL)
return 0;
int endlength = strlen(end);
int strlength = strlen(filename);
if(strlength <= endlength)
return 0;
while (i < endlength) {
if (end[i] != filename[strlength - (endlength - i)])
return 0;
i++;
}
return 1;
}
以上代码是基于opencv框架。
遗留问题
只是实现效果，并未深究细节，存在如下问题
- 贴纸并没有根据人脸倾斜角度而旋转缩放
- 应该加入记忆和基本纠错功能，防止某帧检测不精准而贴图位置出错
- 对于动态贴纸，应该解码贴纸视频获取贴纸的数据，如果贴纸素材较小动画简单也应该使用序列帧代替固定贴纸图片。
