Skip to content

affective_computing / face_recognize_mnn

Go to a project
Toggle navigation
Toggle navigation pinning
ed52fddf by 乔峰昇
Options

submmit

0 parents
Inline Side-by-side
Showing 11 changed files with 797 additions and 0 deletions
CMakeLists.txt 0 → 100644
1 cmake_minimum_required(VERSION 3.10)
2 project(main)
3
4 set(CMAKE_CXX_STANDARD 11)
5
6 find_package(OpenCV REQUIRED)
7 set(MNN_DIR /home/situ/MNN/MNN1.0/MNN)
8 include_directories(${MNN_DIR}/include)
9 LINK_DIRECTORIES(${MNN_DIR}/build)
10 add_executable(main main.cpp faceLandmarks.cpp)
11 # add_executable(main z.cpp)
12 target_link_libraries(main -lMNN ${OpenCV_LIBS})
13
faceLandmarks.cpp 0 → 100644
1 #include "faceLandmarks.h"
2
3
4 vector<vector<float>> FaceLandmarks::detect_landmarks(std::string image_path){
5
6 Mat input_data_=cv::imread(image_path);
7 float w_r=float(input_data_.cols)/112.0f;
8 float h_r=float(input_data_.rows)/112.0f;
9
10 Mat input_data;
11 cv::resize(input_data_,input_data,Size2d(112,112));
12 input_data.convertTo(input_data, CV_32F);
13 input_data = input_data /256.0f;
14 std::vector<std::vector<cv::Mat>> nChannels;
15 std::vector<cv::Mat> rgbChannels(3);
16 cv::split(input_data, rgbChannels);
17 nChannels.push_back(rgbChannels); // NHWC 转NCHW
18 auto *pvData = malloc(1 * 3 * 112 * 112 *sizeof(float));
19 int nPlaneSize = 112 * 112;
20 for (int c = 0; c < 3; ++c)
21 {
22 cv::Mat matPlane = nChannels[0][c];
23 memcpy((float *)(pvData) + c * nPlaneSize,\
24 matPlane.data, nPlaneSize * sizeof(float));
25 }
26 auto inTensor = net->getSessionInput(session, NULL);
27 net->resizeTensor(inTensor, {1, 3, 112,112});
28 net->resizeSession(session);
29 auto nchwTensor = new Tensor(inTensor, Tensor::CAFFE);
30 ::memcpy(nchwTensor->host<float>(), pvData, nPlaneSize * 3 * sizeof(float));
31 inTensor->copyFromHostTensor(nchwTensor);
32 // //推理
33 net->runSession(session);
34 auto output= net->getSessionOutput(session, NULL);
35
36 MNN::Tensor feat_tensor(output, output->getDimensionType());
37 output->copyToHostTensor(&feat_tensor);
38
39 vector<vector<float>> landmarks;
40 for(int idx =0;idx<106;++idx){
41 float x_= *(feat_tensor.host<float>()+2*idx)*w_r;
42 float y_= *(feat_tensor.host<float>()+2*idx+1)*h_r;
43 vector<float> tmp={x_,y_};
44 landmarks.push_back(tmp);
45 }
46 return landmarks;
47 }
48
49 vector<vector<float>> FaceLandmarks::detect_image_landmarks(Mat image){
50
51 Mat input_data_=image;
52 float w_r=float(input_data_.cols)/112.0f;
53 float h_r=float(input_data_.rows)/112.0f;
54
55 Mat input_data;
56 cv::resize(input_data_,input_data,Size2d(112,112));
57 input_data.convertTo(input_data, CV_32F);
58 input_data = input_data /256.0f;
59 std::vector<std::vector<cv::Mat>> nChannels;
60 std::vector<cv::Mat> rgbChannels(3);
61 cv::split(input_data, rgbChannels);
62 nChannels.push_back(rgbChannels); // NHWC 转NCHW
63 auto *pvData = malloc(1 * 3 * 112 * 112 *sizeof(float));
64 int nPlaneSize = 112 * 112;
65 for (int c = 0; c < 3; ++c)
66 {
67 cv::Mat matPlane = nChannels[0][c];
68 memcpy((float *)(pvData) + c * nPlaneSize,\
69 matPlane.data, nPlaneSize * sizeof(float));
70 }
71 auto inTensor = net->getSessionInput(session, NULL);
72 net->resizeTensor(inTensor, {1, 3, 112,112});
73 net->resizeSession(session);
74 auto nchwTensor = new Tensor(inTensor, Tensor::CAFFE);
75 ::memcpy(nchwTensor->host<float>(), pvData, nPlaneSize * 3 * sizeof(float));
76 inTensor->copyFromHostTensor(nchwTensor);
77 // //推理
78 net->runSession(session);
79 auto output= net->getSessionOutput(session, NULL);
80
81 MNN::Tensor feat_tensor(output, output->getDimensionType());
82 output->copyToHostTensor(&feat_tensor);
83
84 vector<vector<float>> landmarks;
85 for(int idx =0;idx<106;++idx){
86 float x_= *(feat_tensor.host<float>()+2*idx)*w_r;
87 float y_= *(feat_tensor.host<float>()+2*idx+1)*h_r;
88 vector<float> tmp={x_,y_};
89 landmarks.push_back(tmp);
90 }
91 return landmarks;
92 }
...\ No newline at end of file ...\ No newline at end of file
faceLandmarks.h 0 → 100644
1 #ifndef FACELANDMARKS_H
2 #define FACELANDMARKS_H
3
4 #include <opencv2/opencv.hpp>
5 #include<MNN/Interpreter.hpp>
6 #include<MNN/ImageProcess.hpp>
7 #include<iostream>
8
9 using namespace std;
10 using namespace cv;
11 using namespace MNN;
12
13 class FaceLandmarks{
14 private:
15 vector<float> input_size={112,112};
16 std::shared_ptr<MNN::Interpreter> net;
17 Session *session = nullptr;
18 ScheduleConfig config;
19
20 public:
21 FaceLandmarks(){};
22 FaceLandmarks(string model_path){
23 net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(model_path.c_str()));//创建解释器
24 config.numThread = 8;
25 config.type = MNN_FORWARD_CPU;
26 session = net->createSession(config);//创建session
27 }
28
29 vector<vector<float>> detect_landmarks(string image_path);
30 vector<vector<float>> detect_image_landmarks(cv::Mat image);
31
32 };
33
34
35 #endif
...\ No newline at end of file ...\ No newline at end of file
facerecognize.cpp 0 → 100644
1 #include "facerecognize.h"
2
3 cv::Mat FaceRecognize::meanAxis0(const cv::Mat &src)
4 {
5 int num = src.rows;
6 int dim = src.cols;
7
8 cv::Mat output(1,dim,CV_32F);
9 for(int i = 0 ; i < dim; i ++)
10 {
11 float sum = 0 ;
12 for(int j = 0 ; j < num ; j++)
13 {
14 sum+=src.at<float>(j,i);
15 }
16 output.at<float>(0,i) = sum/num;
17 }
18
19 return output;
20 }
21
22 cv::Mat FaceRecognize::elementwiseMinus(const cv::Mat &A,const cv::Mat &B)
23 {
24 cv::Mat output(A.rows,A.cols,A.type());
25
26 assert(B.cols == A.cols);
27 if(B.cols == A.cols)
28 {
29 for(int i = 0 ; i < A.rows; i ++)
30 {
31 for(int j = 0 ; j < B.cols; j++)
32 {
33 output.at<float>(i,j) = A.at<float>(i,j) - B.at<float>(0,j);
34 }
35 }
36 }
37 return output;
38 }
39
40 cv::Mat FaceRecognize::varAxis0(const cv::Mat &src)
41 {
42 cv:Mat temp_ = elementwiseMinus(src,meanAxis0(src));
43 cv::multiply(temp_ ,temp_ ,temp_ );
44 return meanAxis0(temp_);
45
46 }
47
48 int FaceRecognize::MatrixRank(cv::Mat M)
49 {
50 Mat w, u, vt;
51 SVD::compute(M, w, u, vt);
52 Mat1b nonZeroSingularValues = w > 0.0001;
53 int rank = countNonZero(nonZeroSingularValues);
54 return rank;
55
56 }
57
58 cv::Mat FaceRecognize::similarTransform(cv::Mat src,cv::Mat dst) {
59 int num = src.rows;
60 int dim = src.cols;
61 cv::Mat src_mean = meanAxis0(src);
62 cv::Mat dst_mean = meanAxis0(dst);
63 cv::Mat src_demean = elementwiseMinus(src, src_mean);
64 cv::Mat dst_demean = elementwiseMinus(dst, dst_mean);
65 cv::Mat A = (dst_demean.t() * src_demean) / static_cast<float>(num);
66 cv::Mat d(dim, 1, CV_32F);
67 d.setTo(1.0f);
68 if (cv::determinant(A) < 0) {
69 d.at<float>(dim - 1, 0) = -1;
70 }
71 Mat T = cv::Mat::eye(dim + 1, dim + 1, CV_32F);
72 cv::Mat U, S, V;
73 SVD::compute(A, S,U, V);
74
75 // the SVD function in opencv differ from scipy .
76 int rank = MatrixRank(A);
77 if (rank == 0) {
78 assert(rank == 0);
79
80 } else if (rank == dim - 1) {
81 if (cv::determinant(U) * cv::determinant(V) > 0) {
82 T.rowRange(0, dim).colRange(0, dim) = U * V;
83 } else {
84 int s = d.at<float>(dim - 1, 0) = -1;
85 d.at<float>(dim - 1, 0) = -1;
86
87 T.rowRange(0, dim).colRange(0, dim) = U * V;
88 cv::Mat diag_ = cv::Mat::diag(d);
89 cv::Mat twp = diag_*V; //np.dot(np.diag(d), V.T)
90 Mat B = Mat::zeros(3, 3, CV_8UC1);
91 Mat C = B.diag(0);
92 T.rowRange(0, dim).colRange(0, dim) = U* twp;
93 d.at<float>(dim - 1, 0) = s;
94 }
95 }
96 else{
97 cv::Mat diag_ = cv::Mat::diag(d);
98 cv::Mat twp = diag_*V.t(); //np.dot(np.diag(d), V.T)
99 cv::Mat res = U* twp; // U
100 T.rowRange(0, dim).colRange(0, dim) = -U.t()* twp;
101 }
102 cv::Mat var_ = varAxis0(src_demean);
103 float val = cv::sum(var_).val[0];
104 cv::Mat res;
105 cv::multiply(d,S,res);
106 float scale = 1.0/val*cv::sum(res).val[0];
107 T.rowRange(0, dim).colRange(0, dim) = - T.rowRange(0, dim).colRange(0, dim).t();
108 cv::Mat temp1 = T.rowRange(0, dim).colRange(0, dim); // T[:dim, :dim]
109 cv::Mat temp2 = src_mean.t(); //src_mean.T
110 cv::Mat temp3 = temp1*temp2; // np.dot(T[:dim, :dim], src_mean.T)
111 cv::Mat temp4 = scale*temp3;
112 T.rowRange(0, dim).colRange(dim, dim+1)= -(temp4 - dst_mean.t()) ;
113 T.rowRange(0, dim).colRange(0, dim) *= scale;
114 return T;
115 }
116
117 Mat FaceRecognize::preprocess_face(Mat image,vector<vector<float>> land){
118 Mat out;
119 cv::resize(image,out,Size(112,112));
120 float default1[5][2] = {
121 {38.2946f, 51.6963f},
122 {73.5318f, 51.6963f},
123 {56.0252f, 71.7366f},
124 {41.5493f, 92.3655f},
125 {70.7299f, 92.3655f}
126 };
127
128 float lands[5][2]={
129 {float(land[0][0]*112.0)/float(image.cols),float(land[0][1]*112.0)/float(image.rows)},
130 {float(land[1][0]*112.0)/float(image.cols),float(land[1][1]*112.0)/float(image.rows)},
131 {float(land[2][0]*112.0)/float(image.cols),float(land[2][1]*112.0)/float(image.rows)},
132 {float(land[3][0]*112.0)/float(image.cols),float(land[3][1]*112.0)/float(image.rows)},
133 {float(land[4][0]*112.0)/float(image.cols),float(land[4][1]*112.0)/float(image.rows)}
134 };
135 cv::Mat src(5,2,CV_32FC1,default1);
136 memcpy(src.data, default1, 2 * 5 * sizeof(float));
137 cv::Mat dst(5,2,CV_32FC1,lands);
138 memcpy(dst.data, lands, 2 * 5 * sizeof(float));
139 cv::Mat M = similarTransform(dst, src);
140 float M_[2][3]={
141 {M.at<float>(0,0),M.at<float>(0,1),M.at<float>(0,2)},
142 {M.at<float>(1,0),M.at<float>(1,1),M.at<float>(1,2)},
143 };
144
145 cv::Mat M__(2,3,CV_32FC1,M_);
146 cv::Mat align_image;
147 cv::warpAffine(out,align_image,M__,Size(112, 112));
148 return align_image;
149 }
150
151 double FaceRecognize::getMold(const vector<double>& vec)
152 {
153 int n = vec.size();
154 double sum = 0.0;
155 for (int i = 0; i < n; ++i)
156 sum += vec[i] * vec[i];
157 return sqrt(sum);
158 }
159
160 double FaceRecognize::cos_distance(const vector<double>& base, const vector<double>& target)
161 {
162 int n = base.size();
163 assert(n == target.size());
164 double tmp = 0.0;
165 for (int i = 0; i < n; ++i)
166 tmp += base[i] * target[i];
167 double simility = tmp / (getMold(base)*getMold(target));
168 return simility;
169 }
170
171 double FaceRecognize::get_samilar(Mat image1,Mat image2){
172 cv::resize(image1,image1,Size2d(input_size[0],input_size[1]));
173 cv::resize(image2,image2,Size2d(input_size[0],input_size[1]));
174 image1.convertTo(image1, CV_32F);
175 image2.convertTo(image2, CV_32F);
176 image1 = (image1-mean)*scale;
177 image2 = (image2-mean)*scale;
178
179 std::vector<std::vector<cv::Mat>> nChannels1;
180 std::vector<cv::Mat> rgbChannels1(3);
181 cv::split(image1, rgbChannels1);
182 nChannels1.push_back(rgbChannels1); // NHWC 转NCHW
183 auto *pvData1 = malloc(1 * 3 * input_size[1] * input_size[0] *sizeof(float));
184 int nPlaneSize = input_size[0] * input_size[1];
185 for (int c = 0; c < 3; ++c)
186 {
187 cv::Mat matPlane1 = nChannels1[0][c];
188 memcpy((float *)(pvData1) + c * nPlaneSize,\
189 matPlane1.data, nPlaneSize * sizeof(float));
190 }
191 auto inTensor1 = net->getSessionInput(session1, NULL);
192 net->resizeTensor(inTensor1, {1, 3, input_size[1],input_size[0]});
193 net->resizeSession(session1);
194
195 auto nchwTensor1 = new Tensor(inTensor1, Tensor::CAFFE);
196 ::memcpy(nchwTensor1->host<float>(), pvData1, nPlaneSize * 3 * sizeof(float));
197 inTensor1->copyFromHostTensor(nchwTensor1);
198 // //推理
199 net->runSession(session1);
200 auto output1= net->getSessionOutput(session1, NULL);
201
202 std::vector<std::vector<cv::Mat>> nChannels2;
203 std::vector<cv::Mat> rgbChannels2(3);
204 cv::split(image2, rgbChannels2);
205 nChannels2.push_back(rgbChannels2); // NHWC 转NCHW
206 auto *pvData2 = malloc(1 * 3 * input_size[1] * input_size[0] *sizeof(float));
207 for (int c = 0; c < 3; ++c)
208 {
209 cv::Mat matPlane2 = nChannels2[0][c];
210 memcpy((float *)(pvData2) + c * nPlaneSize,\
211 matPlane2.data, nPlaneSize * sizeof(float));
212 }
213 auto inTensor2 = net->getSessionInput(session2, NULL);
214 net->resizeTensor(inTensor2, {1, 3, input_size[1],input_size[0]});
215 net->resizeSession(session2);
216 auto nchwTensor2 = new Tensor(inTensor2, Tensor::CAFFE);
217 ::memcpy(nchwTensor2->host<float>(), pvData2, nPlaneSize * 3 * sizeof(float));
218 inTensor2->copyFromHostTensor(nchwTensor2);
219 // //推理
220 net->runSession(session2);
221 auto output2= net->getSessionOutput(session2, NULL);
222
223
224 MNN::Tensor feat_tensor1(output1, MNN::Tensor::CAFFE);
225 MNN::Tensor feat_tensor2(output2, MNN::Tensor::CAFFE);
226 output1->copyToHostTensor(&feat_tensor1);
227 output2->copyToHostTensor(&feat_tensor2);
228 auto feature1 = feat_tensor1.host<float>();
229 auto feature2 = feat_tensor2.host<float>();
230
231 vector<double> v1,v2;
232 for(int i=0;i<int(feat_tensor1.size()/4);i++){
233 v1.push_back((double)feature1[i]);
234 v2.push_back((double)feature2[i]);
235 }
236 double cos_score=cos_distance(v1,v2);
237 return cos_score;
238 }
239
facerecognize.h 0 → 100644
1 #ifndef FACERECOGNIZE_H
2 #define FACERECOGNIZE_H
3 #include<opencv2/opencv.hpp>
4 #include<MNN/Interpreter.hpp>
5 #include<MNN/ImageProcess.hpp>
6 #include<iostream>
7
8 using namespace MNN;
9 using namespace std;
10 using namespace cv;
11 class FaceRecognize{
12 private:
13 vector<float> input_size={112,112};
14 std::shared_ptr<MNN::Interpreter> net;
15 Session *session1 = nullptr;
16 Session *session2 = nullptr;
17 ScheduleConfig config;
18 Scalar mean=Scalar(127.5f,127.5f,127.5f);
19 float scale = 1.0f/127.5f;
20
21 public:
22 FaceRecognize(){};
23 FaceRecognize(string model_path){
24 net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(model_path.c_str()));//创建解释器
25 config.numThread = 8;
26 config.type = MNN_FORWARD_CPU;
27 session1 = net->createSession(config);//创建session
28 session2 = net->createSession(config);//创建session
29 }
30 //预处理
31 cv::Mat meanAxis0(const cv::Mat &src);
32 cv::Mat elementwiseMinus(const cv::Mat &A,const cv::Mat &B);
33 cv::Mat varAxis0(const cv::Mat &src);
34 int MatrixRank(cv::Mat M);
35 cv::Mat similarTransform(cv::Mat src,cv::Mat dst);
36 Mat preprocess_face(Mat image,vector<vector<float>> land);
37 double getMold(const vector<double>& vec);
38 double cos_distance(const vector<double>& base, const vector<double>& target);
39 // 推理
40 double get_samilar(Mat image1,Mat image2);
41 };
42 #endif
...\ No newline at end of file ...\ No newline at end of file
main.cpp 0 → 100644
1 #include "faceLandmarks.h"
2 int main(){
3 FaceLandmarks face_landmarks1 = FaceLandmarks("/home/situ/qfs/sdk_project/gitlab_demo/face_recognize_mnn/model/det_landmarks_106_v0.0.1.mnn");
4 vector<string> filenames;
5 cv::glob("/home/situ/图片/img3", filenames, false);
6 for(auto path:filenames){
7
8 // cout<<path<<endl;
9 Mat img1 =cv::imread(path);
10 auto landmarks1 = face_landmarks1.detect_landmarks(path);
11 for(auto landm:landmarks1){
12 cv::circle(img1,Point2d(landm[0],landm[1]),2,Scalar(255,0,0));
13 }
14 cv::imshow("img",img1);
15 cv::waitKey(0);
16 }
17
18 // Mat image1 = cv::imread("/home/situ/图片/4.jpg");
19 // Mat image2 = cv::imread("/home/situ/图片/img3/1.jpg");
20 // string face_det_model = "/home/situ/qfs/sdk_project/face_recognize_mnn/model/mnn/det_face_retina_mnn_1.0.0_v0.1.1.mnn";
21 // string face_landm_model = "/home/situ/qfs/sdk_project/face_recognize_mnn/model/mnn/det_landmarks_106_v0.0.1.mnn";
22 // string face_rec_model = "/home/situ/qfs/mobile_face_recognize/models/cls_face_mnn_1.0.0_v0.1.0.mnn";
23
24 // FaceComparison face_rec = FaceComparison(face_det_model,face_landm_model,face_rec_model);
25 // bool result = face_rec.face_compare("/home/situ/图片/2.png","/home/situ/图片/2.png");
26 // cout<<result<<endl;
27 }
model/cls_face_mnn_1.0.0_v0.0.2.mnn 0 → 100644
No preview for this file type
model/det_face_retina_mnn_1.0.0_v0.1.1.mnn 0 → 100644
No preview for this file type
model/det_landmarks_106_v0.0.1.mnn 0 → 100644
No preview for this file type
retinaface.cpp 0 → 100644
1 #include "retinaface.h"
2 // 生成anchors
3 vector<vector<float>> RetinaFace::priorBox(vector<float> image_size){
4 vector<int> tmp1={16,32};
5 vector<int> tmp2={64,128};
6 vector<int> tmp3={256,512};
7 vector<vector<int>> min_sizes_;
8 min_sizes_.push_back(tmp1);
9 min_sizes_.push_back(tmp2);
10 min_sizes_.push_back(tmp3);
11 vector<int> steps={8,16,32};
12 vector<vector<int>> feature_maps;
13 vector<vector<float>> anchors;
14 for(int &step:steps){
15 vector<int> tmp(2,0);
16 tmp[0]=ceil(image_size[0]/step);
17 tmp[1]=ceil(image_size[1]/step);
18 feature_maps.push_back(tmp);
19 }
20 for(int k=0;k<feature_maps.size();k++){
21 vector<int> min_sizes=min_sizes_[k];
22
23 for(int i=0;i<feature_maps[k][0];i++){
24 for(int j=0;j<feature_maps[k][1];j++){
25 for(int &min_size:min_sizes){
26 float s_kx=float(min_size)/float(image_size[1]);
27 float s_ky=float(min_size)/float(image_size[0]);
28 float dense_cx=float((float(j)+float(0.5))*steps[k])/float(image_size[1]);
29 float dense_cy=float((float(i)+float(0.5))*steps[k])/float(image_size[1]);
30 vector<float> tmp_anchor={dense_cx,dense_cy,s_kx,s_ky};
31 anchors.push_back(tmp_anchor);
32 }
33 }
34 }
35 }
36 return anchors;
37 }
38
39 // 解析bounding box 包含置信度
40 vector<Bbox> RetinaFace::decode(float *loc,float *score,float *pre,vector<vector<float>> priors,vector<float> variances){
41 vector<float> input_size={640,640};
42 float resize_scale=1.0;
43 vector<Bbox> boxes;
44 for(int i=0;i<priors.size();++i){
45 float b1=priors[i][0]+loc[4*i]*variances[0]*priors[i][2];
46 float b2=priors[i][1]+loc[4*i+1]*variances[0]*priors[i][3];
47 float b3=priors[i][2]*exp(loc[4*i+2]*variances[1]);
48 float b4=priors[i][3]*exp(loc[4*i+3]*variances[1]);
49 b1=b1-b3/float(2);
50 b2=b2-b4/float(2);
51 b3=b3+b1;
52 b4=b4+b2;
53 float l1=priors[i][0]+pre[10*i]*variances[0]*priors[i][2];
54 float l2=priors[i][1]+pre[10*i+1]*variances[0]*priors[i][3];
55 float l3=priors[i][0]+pre[10*i+2]*variances[0]*priors[i][2];
56 float l4=priors[i][1]+pre[10*i+3]*variances[0]*priors[i][3];
57 float l5=priors[i][0]+pre[10*i+4]*variances[0]*priors[i][2];
58 float l6=priors[i][1]+pre[10*i+5]*variances[0]*priors[i][3];
59 float l7=priors[i][0]+pre[10*i+6]*variances[0]*priors[i][2];
60 float l8=priors[i][1]+pre[10*i+7]*variances[0]*priors[i][3];
61 float l9=priors[i][0]+pre[10*i+8]*variances[0]*priors[i][2];
62 float l10=priors[i][1]+pre[10*i+9]*variances[0]*priors[i][3];
63 b1>0?b1:0;
64 b2>0?b2:0;
65 b3>640?640:b3;
66 b4>640?640:b4;
67 Bbox tmp_box={.xmin=b1*input_size[0]/resize_scale,.ymin=b2*input_size[1]/resize_scale,.xmax=b3*input_size[0]/resize_scale,.ymax=b4*input_size[1]/resize_scale,
68 .score=score[2*i+1],.x1=(l1*input_size[0])/resize_scale,.y1=l2*input_size[1]/resize_scale,.x2=l3*input_size[0]/resize_scale,.y2=l4*input_size[1]/resize_scale,
69 .x3=l5*input_size[0]/resize_scale,.y3=l6*input_size[1]/resize_scale,.x4=l7*input_size[0]/resize_scale,.y4=l8*input_size[1]/resize_scale,.x5=l9*input_size[0]/resize_scale,.y5=l10*input_size[1]/resize_scale};
70 boxes.push_back(tmp_box);
71 }
72 return boxes;
73 }
74
75
76
77 //NMS
78 void RetinaFace::nms_cpu(std::vector<Bbox> &bboxes, float threshold){
79 if (bboxes.empty()){
80 return ;
81 }
82 // 1.之前需要按照score排序
83 std::sort(bboxes.begin(), bboxes.end(), [&](Bbox b1, Bbox b2){return b1.score>b2.score;});
84 // 2.先求出所有bbox自己的大小
85 std::vector<float> area(bboxes.size());
86 for (int i=0; i<bboxes.size(); ++i){
87 area[i] = (bboxes[i].xmax - bboxes[i].xmin + 1) * (bboxes[i].ymax - bboxes[i].ymin + 1);
88 }
89 // 3.循环
90 for (int i=0; i<bboxes.size(); ++i){
91 for (int j=i+1; j<bboxes.size(); ){
92 float left = std::max(bboxes[i].xmin, bboxes[j].xmin);
93 float right = std::min(bboxes[i].xmax, bboxes[j].xmax);
94 float top = std::max(bboxes[i].ymin, bboxes[j].ymin);
95 float bottom = std::min(bboxes[i].ymax, bboxes[j].ymax);
96 float width = std::max(right - left + 1, 0.f);
97 float height = std::max(bottom - top + 1, 0.f);
98 float u_area = height * width;
99 float iou = (u_area) / (area[i] + area[j] - u_area);
100 if (iou>=threshold){
101 bboxes.erase(bboxes.begin()+j);
102 area.erase(area.begin()+j);
103 }else{
104 ++j;
105 }
106 }
107 }
108 }
109
110 // 根据阈值筛选
111 vector<Bbox> RetinaFace::select_score(vector<Bbox> bboxes,float threshold,float w_r,float h_r){
112 vector<Bbox> results;
113 for(Bbox &box:bboxes){
114 if (float(box.score)>=threshold){
115 box.xmin=box.xmin/w_r;
116 box.ymin=box.ymin/h_r;
117 box.xmax=box.xmax/w_r;
118 box.ymax=box.ymax/h_r;
119 box.x1=box.x1/w_r;
120 box.y1=box.y1/h_r;
121 box.x2=box.x2/w_r;
122 box.y2=box.y2/h_r;
123 box.x3=box.x3/w_r;
124 box.y3=box.y3/h_r;
125 box.x4=box.x4/w_r;
126 box.y4=box.y4/h_r;
127 box.x5=box.x5/w_r;
128 box.y5=box.y5/h_r;
129 results.push_back(box);
130 }
131 }
132 return results;
133 }
134
135 // 数据后处理
136 vector<Bbox> RetinaFace::bbox_process(vector<Bbox> bboxes,float frame_w,float frame_h){
137 vector<Bbox> result_bboxes;
138 for(Bbox &bbox:bboxes){
139 Bbox new_bbox;
140 float face_w=bbox.xmax-bbox.xmin;
141 float face_h=bbox.ymax-bbox.ymin;
142 new_bbox.xmin=bbox.xmin-face_w*0.15;
143 new_bbox.xmax=bbox.xmax+face_w*0.15;
144 new_bbox.ymin=bbox.ymin;
145 new_bbox.ymax=bbox.ymax+face_h*0.15;
146 new_bbox.xmin=new_bbox.xmin>0?new_bbox.xmin:0;
147 new_bbox.ymin=new_bbox.ymin>0?new_bbox.ymin:0;
148 new_bbox.xmax=new_bbox.xmax>frame_w?frame_w:new_bbox.xmax;
149 new_bbox.ymax=new_bbox.ymax>frame_h?frame_h:new_bbox.ymax;
150 new_bbox.score=bbox.score;
151 new_bbox.x1=bbox.x1>0?bbox.x1:0;
152 new_bbox.y1=bbox.y1>0?bbox.y1:0;
153 new_bbox.x2=bbox.x2>0?bbox.x2:0;
154 new_bbox.y2=bbox.y2>0?bbox.y2:0;
155 new_bbox.x3=bbox.x3>0?bbox.x3:0;
156 new_bbox.y3=bbox.y3>0?bbox.y3:0;
157 new_bbox.x4=bbox.x4>0?bbox.x4:0;
158 new_bbox.y4=bbox.y4>0?bbox.y4:0;
159 new_bbox.x5=bbox.x5>0?bbox.x5:0;
160 new_bbox.y5=bbox.y5>0?bbox.y5:0;
161 result_bboxes.push_back(new_bbox);
162
163 }
164 return result_bboxes;
165 }
166
167
168 // 推理
169 vector<Bbox> RetinaFace::detect(string image_path){
170 Mat image = cv::imread(image_path);
171 float w_r=float(input_size[0])/float(image.cols);
172 float h_r=float(input_size[1])/float(image.rows);
173 Mat input_data;
174 cv::resize(image,input_data,Size(input_size[0],input_size[1]));
175 input_data = input_data-mean;
176 input_data.convertTo(input_data, CV_32F);
177 std::vector<std::vector<cv::Mat>> nChannels;
178 std::vector<cv::Mat> rgbChannels(3);
179 cv::split(input_data, rgbChannels);
180 nChannels.push_back(rgbChannels); // NHWC 转NCHW
181 auto *pvData = malloc(1 * 3 * input_size[1] * input_size[0] *sizeof(float));
182 int nPlaneSize = input_size[0] * input_size[1];
183 for (int c = 0; c < 3; ++c)
184 {
185 cv::Mat matPlane = nChannels[0][c];
186 memcpy((float *)(pvData) + c * nPlaneSize,\
187 matPlane.data, nPlaneSize * sizeof(float));
188 }
189 auto inTensor = net->getSessionInput(session, NULL);
190 net->resizeTensor(inTensor, {1, 3, input_size[1],input_size[0]});
191 net->resizeSession(session);
192 auto nchwTensor = new Tensor(inTensor, Tensor::CAFFE);
193 ::memcpy(nchwTensor->host<float>(), pvData, nPlaneSize * 3 * sizeof(float));
194 inTensor->copyFromHostTensor(nchwTensor);
195 // //推理
196 net->runSession(session);
197 auto output0= net->getSessionOutput(session, "output0");
198 auto output1= net->getSessionOutput(session, "output1");
199 auto output2= net->getSessionOutput(session, "output2");
200 MNN::Tensor feat_tensor0(output0, MNN::Tensor::CAFFE);
201 MNN::Tensor feat_tensor1(output1, MNN::Tensor::CAFFE);
202 MNN::Tensor feat_tensor2(output2, MNN::Tensor::CAFFE);
203 output0->copyToHostTensor(&feat_tensor0);
204 output1->copyToHostTensor(&feat_tensor1);
205 output2->copyToHostTensor(&feat_tensor2);
206 auto loc = feat_tensor0.host<float>();
207 auto score = feat_tensor1.host<float>();
208 auto landm = feat_tensor2.host<float>();
209
210 vector<Bbox> result_boxes = decode(loc,score,landm,anchors,variances);
211 vector<Bbox> results=select_score(result_boxes,confidence_threshold,w_r,h_r);
212
213 nms_cpu(results,nms_threshold);
214 if(is_bbox_process){
215 vector<Bbox> res_bboxes=bbox_process(results,image.cols,image.rows);
216 return res_bboxes;
217
218 }else{
219 return results;
220 }
221 }
222 vector<Bbox> RetinaFace::detect_image(Mat image){
223 float w_r=float(input_size[0])/float(image.cols);
224 float h_r=float(input_size[1])/float(image.rows);
225 Mat input_data;
226 cv::resize(image,input_data,Size(input_size[0],input_size[1]));
227 input_data = input_data-mean;
228 input_data.convertTo(input_data, CV_32F);
229 std::vector<std::vector<cv::Mat>> nChannels;
230 std::vector<cv::Mat> rgbChannels(3);
231 cv::split(input_data, rgbChannels);
232 nChannels.push_back(rgbChannels); // NHWC 转NCHW
233 auto *pvData = malloc(1 * 3 * input_size[1] * input_size[0] *sizeof(float));
234 int nPlaneSize = input_size[0] * input_size[1];
235 for (int c = 0; c < 3; ++c)
236 {
237 cv::Mat matPlane = nChannels[0][c];
238 memcpy((float *)(pvData) + c * nPlaneSize,\
239 matPlane.data, nPlaneSize * sizeof(float));
240 }
241 auto inTensor = net->getSessionInput(session, NULL);
242 net->resizeTensor(inTensor, {1, 3, input_size[1],input_size[0]});
243 net->resizeSession(session);
244 auto nchwTensor = new Tensor(inTensor, Tensor::CAFFE);
245 ::memcpy(nchwTensor->host<float>(), pvData, nPlaneSize * 3 * sizeof(float));
246 inTensor->copyFromHostTensor(nchwTensor);
247 // //推理
248 net->runSession(session);
249 auto output0= net->getSessionOutput(session, "output0");
250 auto output1= net->getSessionOutput(session, "output1");
251 auto output2= net->getSessionOutput(session, "output2");
252 MNN::Tensor feat_tensor0(output0, MNN::Tensor::CAFFE);
253 MNN::Tensor feat_tensor1(output1, MNN::Tensor::CAFFE);
254 MNN::Tensor feat_tensor2(output2, MNN::Tensor::CAFFE);
255 output0->copyToHostTensor(&feat_tensor0);
256 output1->copyToHostTensor(&feat_tensor1);
257 output2->copyToHostTensor(&feat_tensor2);
258 auto loc = feat_tensor0.host<float>();
259 auto score = feat_tensor1.host<float>();
260 auto landm = feat_tensor2.host<float>();
261
262 vector<Bbox> result_boxes = decode(loc,score,landm,anchors,variances);
263 vector<Bbox> results=select_score(result_boxes,confidence_threshold,w_r,h_r);
264
265 nms_cpu(results,nms_threshold);
266 if(is_bbox_process){
267 vector<Bbox> res_bboxes=bbox_process(results,image.cols,image.rows);
268 return res_bboxes;
269
270 }else{
271 return results;
272 }
273 }
...\ No newline at end of file ...\ No newline at end of file
retinaface.h 0 → 100644
1 #ifndef RETINAFACE_H
2 #define RETINAFACE_H
3 #include<opencv2/opencv.hpp>
4 #include<MNN/Interpreter.hpp>
5 #include<MNN/ImageProcess.hpp>
6 #include<iostream>
7
8 using namespace MNN;
9 using namespace std;
10 using namespace cv;
11 struct Bbox{
12 float xmin;
13 float ymin;
14 float xmax;
15 float ymax;
16 float score;
17 float x1;
18 float y1;
19 float x2;
20 float y2;
21 float x3;
22 float y3;
23 float x4;
24 float y4;
25 float x5;
26 float y5;
27 };
28 class RetinaFace{
29 public:
30 float confidence_threshold = 0.5;
31 bool is_bbox_process=true;
32
33 private:
34 bool use_gpu=true;
35 vector<float> input_size={640,640};
36 vector<float> variances={0.1,0.2};
37 Scalar mean = Scalar(104.0f, 117.0f, 123.0f);
38 float keep_top_k = 100;
39 float nms_threshold = 0.4;
40 float resize_scale = 1.0;
41
42 std::shared_ptr<MNN::Interpreter> net;
43 Session *session = nullptr;
44 ScheduleConfig config;
45 vector<vector<float>> anchors;
46
47 private:
48 // 生成anchors
49 vector<vector<float>> priorBox(vector<float> image_size);
50 // 解析bounding box landmarks 包含置信度
51 vector<Bbox> decode(float *loc,float *score,float *pre,vector<vector<float>> priors,vector<float> variances);
52 // 解析landmarks
53 // vector<vector<float>> decode_landm(vector<vector<float>> pre,vector<vector<float>> priors,vector<float> variances);
54 //NMS
55 void nms_cpu(std::vector<Bbox> &bboxes, float threshold);
56 // 根据阈值筛选
57 vector<Bbox> select_score(vector<Bbox> bboxes,float threshold,float w_r,float h_r);
58 // 数据后处理
59 vector<Bbox> bbox_process(vector<Bbox> bboxes,float frame_w,float frame_h);
60
61 public:
62
63 RetinaFace(){};
64 RetinaFace(string model_path){
65 net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(model_path.c_str()));//创建解释器
66 config.numThread = 8;
67 config.type = MNN_FORWARD_CPU;
68 session = net->createSession(config);//创建session
69 anchors=priorBox(input_size);
70 }
71
72 // 推理
73 vector<Bbox> detect(string image_path);
74 vector<Bbox> detect_image(Mat image);
75 };
76 #endif
...\ No newline at end of file ...\ No newline at end of file
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!