Skip to content

affective_computing / face_recognize_mnn

Go to a project
Toggle navigation
Toggle navigation pinning
ed52fddf by 乔峰昇
Options

submmit

0 parents
Expand all
Inline Side-by-side
Showing 11 changed files with 524 additions and 0 deletions
CMakeLists.txt 0 → 100644
1 cmake_minimum_required(VERSION 3.10)
2 project(main)
3
4 set(CMAKE_CXX_STANDARD 11)
5
6 find_package(OpenCV REQUIRED)
7 set(MNN_DIR /home/situ/MNN/MNN1.0/MNN)
8 include_directories(${MNN_DIR}/include)
9 LINK_DIRECTORIES(${MNN_DIR}/build)
10 add_executable(main main.cpp faceLandmarks.cpp)
11 # add_executable(main z.cpp)
12 target_link_libraries(main -lMNN ${OpenCV_LIBS})
13
faceLandmarks.cpp 0 → 100644
1 #include "faceLandmarks.h"
2
3
4 vector<vector<float>> FaceLandmarks::detect_landmarks(std::string image_path){
5
6 Mat input_data_=cv::imread(image_path);
7 float w_r=float(input_data_.cols)/112.0f;
8 float h_r=float(input_data_.rows)/112.0f;
9
10 Mat input_data;
11 cv::resize(input_data_,input_data,Size2d(112,112));
12 input_data.convertTo(input_data, CV_32F);
13 input_data = input_data /256.0f;
14 std::vector<std::vector<cv::Mat>> nChannels;
15 std::vector<cv::Mat> rgbChannels(3);
16 cv::split(input_data, rgbChannels);
17 nChannels.push_back(rgbChannels); // NHWC 转NCHW
18 auto *pvData = malloc(1 * 3 * 112 * 112 *sizeof(float));
19 int nPlaneSize = 112 * 112;
20 for (int c = 0; c < 3; ++c)
21 {
22 cv::Mat matPlane = nChannels[0][c];
23 memcpy((float *)(pvData) + c * nPlaneSize,\
24 matPlane.data, nPlaneSize * sizeof(float));
25 }
26 auto inTensor = net->getSessionInput(session, NULL);
27 net->resizeTensor(inTensor, {1, 3, 112,112});
28 net->resizeSession(session);
29 auto nchwTensor = new Tensor(inTensor, Tensor::CAFFE);
30 ::memcpy(nchwTensor->host<float>(), pvData, nPlaneSize * 3 * sizeof(float));
31 inTensor->copyFromHostTensor(nchwTensor);
32 // //推理
33 net->runSession(session);
34 auto output= net->getSessionOutput(session, NULL);
35
36 MNN::Tensor feat_tensor(output, output->getDimensionType());
37 output->copyToHostTensor(&feat_tensor);
38
39 vector<vector<float>> landmarks;
40 for(int idx =0;idx<106;++idx){
41 float x_= *(feat_tensor.host<float>()+2*idx)*w_r;
42 float y_= *(feat_tensor.host<float>()+2*idx+1)*h_r;
43 vector<float> tmp={x_,y_};
44 landmarks.push_back(tmp);
45 }
46 return landmarks;
47 }
48
49 vector<vector<float>> FaceLandmarks::detect_image_landmarks(Mat image){
50
51 Mat input_data_=image;
52 float w_r=float(input_data_.cols)/112.0f;
53 float h_r=float(input_data_.rows)/112.0f;
54
55 Mat input_data;
56 cv::resize(input_data_,input_data,Size2d(112,112));
57 input_data.convertTo(input_data, CV_32F);
58 input_data = input_data /256.0f;
59 std::vector<std::vector<cv::Mat>> nChannels;
60 std::vector<cv::Mat> rgbChannels(3);
61 cv::split(input_data, rgbChannels);
62 nChannels.push_back(rgbChannels); // NHWC 转NCHW
63 auto *pvData = malloc(1 * 3 * 112 * 112 *sizeof(float));
64 int nPlaneSize = 112 * 112;
65 for (int c = 0; c < 3; ++c)
66 {
67 cv::Mat matPlane = nChannels[0][c];
68 memcpy((float *)(pvData) + c * nPlaneSize,\
69 matPlane.data, nPlaneSize * sizeof(float));
70 }
71 auto inTensor = net->getSessionInput(session, NULL);
72 net->resizeTensor(inTensor, {1, 3, 112,112});
73 net->resizeSession(session);
74 auto nchwTensor = new Tensor(inTensor, Tensor::CAFFE);
75 ::memcpy(nchwTensor->host<float>(), pvData, nPlaneSize * 3 * sizeof(float));
76 inTensor->copyFromHostTensor(nchwTensor);
77 // //推理
78 net->runSession(session);
79 auto output= net->getSessionOutput(session, NULL);
80
81 MNN::Tensor feat_tensor(output, output->getDimensionType());
82 output->copyToHostTensor(&feat_tensor);
83
84 vector<vector<float>> landmarks;
85 for(int idx =0;idx<106;++idx){
86 float x_= *(feat_tensor.host<float>()+2*idx)*w_r;
87 float y_= *(feat_tensor.host<float>()+2*idx+1)*h_r;
88 vector<float> tmp={x_,y_};
89 landmarks.push_back(tmp);
90 }
91 return landmarks;
92 }
...\ No newline at end of file ...\ No newline at end of file
faceLandmarks.h 0 → 100644
1 #ifndef FACELANDMARKS_H
2 #define FACELANDMARKS_H
3
4 #include <opencv2/opencv.hpp>
5 #include<MNN/Interpreter.hpp>
6 #include<MNN/ImageProcess.hpp>
7 #include<iostream>
8
9 using namespace std;
10 using namespace cv;
11 using namespace MNN;
12
13 class FaceLandmarks{
14 private:
15 vector<float> input_size={112,112};
16 std::shared_ptr<MNN::Interpreter> net;
17 Session *session = nullptr;
18 ScheduleConfig config;
19
20 public:
21 FaceLandmarks(){};
22 FaceLandmarks(string model_path){
23 net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(model_path.c_str()));//创建解释器
24 config.numThread = 8;
25 config.type = MNN_FORWARD_CPU;
26 session = net->createSession(config);//创建session
27 }
28
29 vector<vector<float>> detect_landmarks(string image_path);
30 vector<vector<float>> detect_image_landmarks(cv::Mat image);
31
32 };
33
34
35 #endif
...\ No newline at end of file ...\ No newline at end of file
facerecognize.cpp 0 → 100644
1 #include "facerecognize.h"
2
3 cv::Mat FaceRecognize::meanAxis0(const cv::Mat &src)
4 {
5 int num = src.rows;
6 int dim = src.cols;
7
8 cv::Mat output(1,dim,CV_32F);
9 for(int i = 0 ; i < dim; i ++)
10 {
11 float sum = 0 ;
12 for(int j = 0 ; j < num ; j++)
13 {
14 sum+=src.at<float>(j,i);
15 }
16 output.at<float>(0,i) = sum/num;
17 }
18
19 return output;
20 }
21
22 cv::Mat FaceRecognize::elementwiseMinus(const cv::Mat &A,const cv::Mat &B)
23 {
24 cv::Mat output(A.rows,A.cols,A.type());
25
26 assert(B.cols == A.cols);
27 if(B.cols == A.cols)
28 {
29 for(int i = 0 ; i < A.rows; i ++)
30 {
31 for(int j = 0 ; j < B.cols; j++)
32 {
33 output.at<float>(i,j) = A.at<float>(i,j) - B.at<float>(0,j);
34 }
35 }
36 }
37 return output;
38 }
39
40 cv::Mat FaceRecognize::varAxis0(const cv::Mat &src)
41 {
42 cv:Mat temp_ = elementwiseMinus(src,meanAxis0(src));
43 cv::multiply(temp_ ,temp_ ,temp_ );
44 return meanAxis0(temp_);
45
46 }
47
48 int FaceRecognize::MatrixRank(cv::Mat M)
49 {
50 Mat w, u, vt;
51 SVD::compute(M, w, u, vt);
52 Mat1b nonZeroSingularValues = w > 0.0001;
53 int rank = countNonZero(nonZeroSingularValues);
54 return rank;
55
56 }
57
58 cv::Mat FaceRecognize::similarTransform(cv::Mat src,cv::Mat dst) {
59 int num = src.rows;
60 int dim = src.cols;
61 cv::Mat src_mean = meanAxis0(src);
62 cv::Mat dst_mean = meanAxis0(dst);
63 cv::Mat src_demean = elementwiseMinus(src, src_mean);
64 cv::Mat dst_demean = elementwiseMinus(dst, dst_mean);
65 cv::Mat A = (dst_demean.t() * src_demean) / static_cast<float>(num);
66 cv::Mat d(dim, 1, CV_32F);
67 d.setTo(1.0f);
68 if (cv::determinant(A) < 0) {
69 d.at<float>(dim - 1, 0) = -1;
70 }
71 Mat T = cv::Mat::eye(dim + 1, dim + 1, CV_32F);
72 cv::Mat U, S, V;
73 SVD::compute(A, S,U, V);
74
75 // the SVD function in opencv differ from scipy .
76 int rank = MatrixRank(A);
77 if (rank == 0) {
78 assert(rank == 0);
79
80 } else if (rank == dim - 1) {
81 if (cv::determinant(U) * cv::determinant(V) > 0) {
82 T.rowRange(0, dim).colRange(0, dim) = U * V;
83 } else {
84 int s = d.at<float>(dim - 1, 0) = -1;
85 d.at<float>(dim - 1, 0) = -1;
86
87 T.rowRange(0, dim).colRange(0, dim) = U * V;
88 cv::Mat diag_ = cv::Mat::diag(d);
89 cv::Mat twp = diag_*V; //np.dot(np.diag(d), V.T)
90 Mat B = Mat::zeros(3, 3, CV_8UC1);
91 Mat C = B.diag(0);
92 T.rowRange(0, dim).colRange(0, dim) = U* twp;
93 d.at<float>(dim - 1, 0) = s;
94 }
95 }
96 else{
97 cv::Mat diag_ = cv::Mat::diag(d);
98 cv::Mat twp = diag_*V.t(); //np.dot(np.diag(d), V.T)
99 cv::Mat res = U* twp; // U
100 T.rowRange(0, dim).colRange(0, dim) = -U.t()* twp;
101 }
102 cv::Mat var_ = varAxis0(src_demean);
103 float val = cv::sum(var_).val[0];
104 cv::Mat res;
105 cv::multiply(d,S,res);
106 float scale = 1.0/val*cv::sum(res).val[0];
107 T.rowRange(0, dim).colRange(0, dim) = - T.rowRange(0, dim).colRange(0, dim).t();
108 cv::Mat temp1 = T.rowRange(0, dim).colRange(0, dim); // T[:dim, :dim]
109 cv::Mat temp2 = src_mean.t(); //src_mean.T
110 cv::Mat temp3 = temp1*temp2; // np.dot(T[:dim, :dim], src_mean.T)
111 cv::Mat temp4 = scale*temp3;
112 T.rowRange(0, dim).colRange(dim, dim+1)= -(temp4 - dst_mean.t()) ;
113 T.rowRange(0, dim).colRange(0, dim) *= scale;
114 return T;
115 }
116
117 Mat FaceRecognize::preprocess_face(Mat image,vector<vector<float>> land){
118 Mat out;
119 cv::resize(image,out,Size(112,112));
120 float default1[5][2] = {
121 {38.2946f, 51.6963f},
122 {73.5318f, 51.6963f},
123 {56.0252f, 71.7366f},
124 {41.5493f, 92.3655f},
125 {70.7299f, 92.3655f}
126 };
127
128 float lands[5][2]={
129 {float(land[0][0]*112.0)/float(image.cols),float(land[0][1]*112.0)/float(image.rows)},
130 {float(land[1][0]*112.0)/float(image.cols),float(land[1][1]*112.0)/float(image.rows)},
131 {float(land[2][0]*112.0)/float(image.cols),float(land[2][1]*112.0)/float(image.rows)},
132 {float(land[3][0]*112.0)/float(image.cols),float(land[3][1]*112.0)/float(image.rows)},
133 {float(land[4][0]*112.0)/float(image.cols),float(land[4][1]*112.0)/float(image.rows)}
134 };
135 cv::Mat src(5,2,CV_32FC1,default1);
136 memcpy(src.data, default1, 2 * 5 * sizeof(float));
137 cv::Mat dst(5,2,CV_32FC1,lands);
138 memcpy(dst.data, lands, 2 * 5 * sizeof(float));
139 cv::Mat M = similarTransform(dst, src);
140 float M_[2][3]={
141 {M.at<float>(0,0),M.at<float>(0,1),M.at<float>(0,2)},
142 {M.at<float>(1,0),M.at<float>(1,1),M.at<float>(1,2)},
143 };
144
145 cv::Mat M__(2,3,CV_32FC1,M_);
146 cv::Mat align_image;
147 cv::warpAffine(out,align_image,M__,Size(112, 112));
148 return align_image;
149 }
150
151 double FaceRecognize::getMold(const vector<double>& vec)
152 {
153 int n = vec.size();
154 double sum = 0.0;
155 for (int i = 0; i < n; ++i)
156 sum += vec[i] * vec[i];
157 return sqrt(sum);
158 }
159
160 double FaceRecognize::cos_distance(const vector<double>& base, const vector<double>& target)
161 {
162 int n = base.size();
163 assert(n == target.size());
164 double tmp = 0.0;
165 for (int i = 0; i < n; ++i)
166 tmp += base[i] * target[i];
167 double simility = tmp / (getMold(base)*getMold(target));
168 return simility;
169 }
170
171 double FaceRecognize::get_samilar(Mat image1,Mat image2){
172 cv::resize(image1,image1,Size2d(input_size[0],input_size[1]));
173 cv::resize(image2,image2,Size2d(input_size[0],input_size[1]));
174 image1.convertTo(image1, CV_32F);
175 image2.convertTo(image2, CV_32F);
176 image1 = (image1-mean)*scale;
177 image2 = (image2-mean)*scale;
178
179 std::vector<std::vector<cv::Mat>> nChannels1;
180 std::vector<cv::Mat> rgbChannels1(3);
181 cv::split(image1, rgbChannels1);
182 nChannels1.push_back(rgbChannels1); // NHWC 转NCHW
183 auto *pvData1 = malloc(1 * 3 * input_size[1] * input_size[0] *sizeof(float));
184 int nPlaneSize = input_size[0] * input_size[1];
185 for (int c = 0; c < 3; ++c)
186 {
187 cv::Mat matPlane1 = nChannels1[0][c];
188 memcpy((float *)(pvData1) + c * nPlaneSize,\
189 matPlane1.data, nPlaneSize * sizeof(float));
190 }
191 auto inTensor1 = net->getSessionInput(session1, NULL);
192 net->resizeTensor(inTensor1, {1, 3, input_size[1],input_size[0]});
193 net->resizeSession(session1);
194
195 auto nchwTensor1 = new Tensor(inTensor1, Tensor::CAFFE);
196 ::memcpy(nchwTensor1->host<float>(), pvData1, nPlaneSize * 3 * sizeof(float));
197 inTensor1->copyFromHostTensor(nchwTensor1);
198 // //推理
199 net->runSession(session1);
200 auto output1= net->getSessionOutput(session1, NULL);
201
202 std::vector<std::vector<cv::Mat>> nChannels2;
203 std::vector<cv::Mat> rgbChannels2(3);
204 cv::split(image2, rgbChannels2);
205 nChannels2.push_back(rgbChannels2); // NHWC 转NCHW
206 auto *pvData2 = malloc(1 * 3 * input_size[1] * input_size[0] *sizeof(float));
207 for (int c = 0; c < 3; ++c)
208 {
209 cv::Mat matPlane2 = nChannels2[0][c];
210 memcpy((float *)(pvData2) + c * nPlaneSize,\
211 matPlane2.data, nPlaneSize * sizeof(float));
212 }
213 auto inTensor2 = net->getSessionInput(session2, NULL);
214 net->resizeTensor(inTensor2, {1, 3, input_size[1],input_size[0]});
215 net->resizeSession(session2);
216 auto nchwTensor2 = new Tensor(inTensor2, Tensor::CAFFE);
217 ::memcpy(nchwTensor2->host<float>(), pvData2, nPlaneSize * 3 * sizeof(float));
218 inTensor2->copyFromHostTensor(nchwTensor2);
219 // //推理
220 net->runSession(session2);
221 auto output2= net->getSessionOutput(session2, NULL);
222
223
224 MNN::Tensor feat_tensor1(output1, MNN::Tensor::CAFFE);
225 MNN::Tensor feat_tensor2(output2, MNN::Tensor::CAFFE);
226 output1->copyToHostTensor(&feat_tensor1);
227 output2->copyToHostTensor(&feat_tensor2);
228 auto feature1 = feat_tensor1.host<float>();
229 auto feature2 = feat_tensor2.host<float>();
230
231 vector<double> v1,v2;
232 for(int i=0;i<int(feat_tensor1.size()/4);i++){
233 v1.push_back((double)feature1[i]);
234 v2.push_back((double)feature2[i]);
235 }
236 double cos_score=cos_distance(v1,v2);
237 return cos_score;
238 }
239
facerecognize.h 0 → 100644
1 #ifndef FACERECOGNIZE_H
2 #define FACERECOGNIZE_H
3 #include<opencv2/opencv.hpp>
4 #include<MNN/Interpreter.hpp>
5 #include<MNN/ImageProcess.hpp>
6 #include<iostream>
7
8 using namespace MNN;
9 using namespace std;
10 using namespace cv;
11 class FaceRecognize{
12 private:
13 vector<float> input_size={112,112};
14 std::shared_ptr<MNN::Interpreter> net;
15 Session *session1 = nullptr;
16 Session *session2 = nullptr;
17 ScheduleConfig config;
18 Scalar mean=Scalar(127.5f,127.5f,127.5f);
19 float scale = 1.0f/127.5f;
20
21 public:
22 FaceRecognize(){};
23 FaceRecognize(string model_path){
24 net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(model_path.c_str()));//创建解释器
25 config.numThread = 8;
26 config.type = MNN_FORWARD_CPU;
27 session1 = net->createSession(config);//创建session
28 session2 = net->createSession(config);//创建session
29 }
30 //预处理
31 cv::Mat meanAxis0(const cv::Mat &src);
32 cv::Mat elementwiseMinus(const cv::Mat &A,const cv::Mat &B);
33 cv::Mat varAxis0(const cv::Mat &src);
34 int MatrixRank(cv::Mat M);
35 cv::Mat similarTransform(cv::Mat src,cv::Mat dst);
36 Mat preprocess_face(Mat image,vector<vector<float>> land);
37 double getMold(const vector<double>& vec);
38 double cos_distance(const vector<double>& base, const vector<double>& target);
39 // 推理
40 double get_samilar(Mat image1,Mat image2);
41 };
42 #endif
...\ No newline at end of file ...\ No newline at end of file
main.cpp 0 → 100644
1 #include "faceLandmarks.h"
2 int main(){
3 FaceLandmarks face_landmarks1 = FaceLandmarks("/home/situ/qfs/sdk_project/gitlab_demo/face_recognize_mnn/model/det_landmarks_106_v0.0.1.mnn");
4 vector<string> filenames;
5 cv::glob("/home/situ/图片/img3", filenames, false);
6 for(auto path:filenames){
7
8 // cout<<path<<endl;
9 Mat img1 =cv::imread(path);
10 auto landmarks1 = face_landmarks1.detect_landmarks(path);
11 for(auto landm:landmarks1){
12 cv::circle(img1,Point2d(landm[0],landm[1]),2,Scalar(255,0,0));
13 }
14 cv::imshow("img",img1);
15 cv::waitKey(0);
16 }
17
18 // Mat image1 = cv::imread("/home/situ/图片/4.jpg");
19 // Mat image2 = cv::imread("/home/situ/图片/img3/1.jpg");
20 // string face_det_model = "/home/situ/qfs/sdk_project/face_recognize_mnn/model/mnn/det_face_retina_mnn_1.0.0_v0.1.1.mnn";
21 // string face_landm_model = "/home/situ/qfs/sdk_project/face_recognize_mnn/model/mnn/det_landmarks_106_v0.0.1.mnn";
22 // string face_rec_model = "/home/situ/qfs/mobile_face_recognize/models/cls_face_mnn_1.0.0_v0.1.0.mnn";
23
24 // FaceComparison face_rec = FaceComparison(face_det_model,face_landm_model,face_rec_model);
25 // bool result = face_rec.face_compare("/home/situ/图片/2.png","/home/situ/图片/2.png");
26 // cout<<result<<endl;
27 }
model/cls_face_mnn_1.0.0_v0.0.2.mnn 0 → 100644
No preview for this file type
model/det_face_retina_mnn_1.0.0_v0.1.1.mnn 0 → 100644
No preview for this file type
model/det_landmarks_106_v0.0.1.mnn 0 → 100644
No preview for this file type
retinaface.h 0 → 100644
1 #ifndef RETINAFACE_H
2 #define RETINAFACE_H
3 #include<opencv2/opencv.hpp>
4 #include<MNN/Interpreter.hpp>
5 #include<MNN/ImageProcess.hpp>
6 #include<iostream>
7
8 using namespace MNN;
9 using namespace std;
10 using namespace cv;
11 struct Bbox{
12 float xmin;
13 float ymin;
14 float xmax;
15 float ymax;
16 float score;
17 float x1;
18 float y1;
19 float x2;
20 float y2;
21 float x3;
22 float y3;
23 float x4;
24 float y4;
25 float x5;
26 float y5;
27 };
28 class RetinaFace{
29 public:
30 float confidence_threshold = 0.5;
31 bool is_bbox_process=true;
32
33 private:
34 bool use_gpu=true;
35 vector<float> input_size={640,640};
36 vector<float> variances={0.1,0.2};
37 Scalar mean = Scalar(104.0f, 117.0f, 123.0f);
38 float keep_top_k = 100;
39 float nms_threshold = 0.4;
40 float resize_scale = 1.0;
41
42 std::shared_ptr<MNN::Interpreter> net;
43 Session *session = nullptr;
44 ScheduleConfig config;
45 vector<vector<float>> anchors;
46
47 private:
48 // 生成anchors
49 vector<vector<float>> priorBox(vector<float> image_size);
50 // 解析bounding box landmarks 包含置信度
51 vector<Bbox> decode(float *loc,float *score,float *pre,vector<vector<float>> priors,vector<float> variances);
52 // 解析landmarks
53 // vector<vector<float>> decode_landm(vector<vector<float>> pre,vector<vector<float>> priors,vector<float> variances);
54 //NMS
55 void nms_cpu(std::vector<Bbox> &bboxes, float threshold);
56 // 根据阈值筛选
57 vector<Bbox> select_score(vector<Bbox> bboxes,float threshold,float w_r,float h_r);
58 // 数据后处理
59 vector<Bbox> bbox_process(vector<Bbox> bboxes,float frame_w,float frame_h);
60
61 public:
62
63 RetinaFace(){};
64 RetinaFace(string model_path){
65 net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(model_path.c_str()));//创建解释器
66 config.numThread = 8;
67 config.type = MNN_FORWARD_CPU;
68 session = net->createSession(config);//创建session
69 anchors=priorBox(input_size);
70 }
71
72 // 推理
73 vector<Bbox> detect(string image_path);
74 vector<Bbox> detect_image(Mat image);
75 };
76 #endif
...\ No newline at end of file ...\ No newline at end of file
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!