Merge branch 'feature/ltgt' into feature/0611
Showing
5 changed files
with
569 additions
and
31 deletions
... | @@ -13,6 +13,9 @@ class OCR2Exception(Exception): | ... | @@ -13,6 +13,9 @@ class OCR2Exception(Exception): |
13 | class OCR4Exception(Exception): | 13 | class OCR4Exception(Exception): |
14 | pass | 14 | pass |
15 | 15 | ||
16 | class LTGTException(Exception): | ||
17 | pass | ||
18 | |||
16 | 19 | ||
17 | class GCAPException(Exception): | 20 | class GCAPException(Exception): |
18 | pass | 21 | pass | ... | ... |
1 | import os | ||
2 | import re | ||
3 | import time | ||
4 | import shutil | ||
5 | import base64 | ||
6 | import signal | ||
7 | import requests | ||
8 | import traceback | ||
9 | from PIL import Image | ||
10 | from datetime import datetime | ||
11 | from django.core.management import BaseCommand | ||
12 | from multiprocessing import Process, Queue | ||
13 | from openpyxl import load_workbook, Workbook | ||
14 | |||
15 | from settings import conf | ||
16 | from common.mixins import LoggerMixin | ||
17 | from common.tools.pdf_to_img import PDFHandler | ||
18 | from apps.doc import consts | ||
19 | from apps.doc.exceptions import OCR1Exception, OCR4Exception, LTGTException | ||
20 | from apps.doc.ocr.wb import BSWorkbook | ||
21 | |||
22 | |||
23 | class TIFFHandler: | ||
24 | |||
25 | def __init__(self, path, img_save_path): | ||
26 | self.path = path | ||
27 | self.img_save_path = img_save_path | ||
28 | self.img_path_list = [] | ||
29 | |||
30 | def extract_image(self): | ||
31 | os.makedirs(self.img_save_path, exist_ok=True) | ||
32 | tiff = Image.open(self.path) | ||
33 | tiff.load() | ||
34 | |||
35 | for i in range(tiff.n_frames): | ||
36 | try: | ||
37 | save_path = os.path.join(self.img_save_path, 'page_{0}.jpeg'.format(i)) | ||
38 | tiff.seek(i) | ||
39 | tiff.save(save_path) | ||
40 | self.img_path_list.append(save_path) | ||
41 | except EOFError: | ||
42 | break | ||
43 | |||
44 | |||
45 | class Command(BaseCommand, LoggerMixin): | ||
46 | |||
47 | def __init__(self): | ||
48 | super().__init__() | ||
49 | self.log_base = '[folder ltgt process]' | ||
50 | # 处理文件开关 | ||
51 | self.switch = True | ||
52 | self.ltgt_classify_mapping = { | ||
53 | 128: '执行裁定书', | ||
54 | 129: '民事判决书', | ||
55 | 130: '民事调解书' | ||
56 | } | ||
57 | self.sheet_content = { | ||
58 | 128: ('执行裁定书', ('承办法院', '案号/标号', '被执行人', '债权金额', '诉讼时间')), | ||
59 | 129: ('民事判决书', ('承办法院', '案号/标号', '被告', '判决结果: 贷款本金', '判决结果: 罚息', '判决结果: 律师费', '判决结果: 案件受理费', '诉讼时间')), | ||
60 | 130: ('民事调解书', ('承办法院', '案号/标号', '被告', '协议内容: 支付金额', '协议内容: 案件受理费', '诉讼时间')), | ||
61 | } | ||
62 | self.DATE_KEY = 'date' | ||
63 | self.CLASSIFY_KEY = 'classify' | ||
64 | self.RESULT_KEY = 'result' | ||
65 | self.daily_wb_name = 'Output_{0}.xlsx' | ||
66 | self.short_sleep_time = 10 | ||
67 | self.long_sleep_time = 3600 | ||
68 | # 睡眠时间 | ||
69 | self.sleep_time = float(conf.SLEEP_SECOND_FOLDER) | ||
70 | # input folder | ||
71 | self.input_dirs = conf.get_namespace('LTGT_DIR_') | ||
72 | # ocr相关 | ||
73 | # self.ocr_url = conf.OCR_URL_FOLDER | ||
74 | # self.ocr_url_4 = conf.IC_URL | ||
75 | self.ltgt_ocr_url = conf.LTGT_URL | ||
76 | # 优雅退出信号:15 | ||
77 | signal.signal(signal.SIGTERM, self.signal_handler) | ||
78 | |||
79 | def signal_handler(self, sig, frame): | ||
80 | self.switch = False # 停止处理文件 | ||
81 | |||
82 | def license1_process(self, ocr_data, license_summary, classify, img_path): | ||
83 | # 类别:'0'身份证, '1'居住证 | ||
84 | license_data = ocr_data.get('data', []) | ||
85 | if not license_data: | ||
86 | return | ||
87 | if classify == consts.MVC_CLASSIFY: # 车辆登记证 3/4页结果整合 | ||
88 | for mvc_dict in license_data: | ||
89 | try: | ||
90 | mvc_page = mvc_dict.pop('page') | ||
91 | except Exception as e: | ||
92 | pass | ||
93 | else: | ||
94 | if mvc_page == 'VehicleRegArea': | ||
95 | mvc_res = mvc_dict.pop('results', {}) | ||
96 | mvc_dict['机动车登记证书编号'] = mvc_res.get('register_no', {}).get('words', '') | ||
97 | for register_info in mvc_res.get('register_info', []): | ||
98 | for detail_dict in register_info.get('details', {}).values(): | ||
99 | mvc_dict.setdefault(detail_dict.get('chinese_key', '未知'), []).append( | ||
100 | detail_dict.get('words', '')) | ||
101 | del mvc_res | ||
102 | if classify == consts.IC_CLASSIFY: | ||
103 | for id_card_dict in license_data: | ||
104 | try: | ||
105 | base64_img = id_card_dict.pop('base64_img') | ||
106 | except Exception as e: | ||
107 | continue | ||
108 | else: | ||
109 | card_type = -1 | ||
110 | json_data_4 = { | ||
111 | 'mode': 1, | ||
112 | 'user_info': { | ||
113 | 'image_content': base64_img, | ||
114 | }, | ||
115 | 'options': { | ||
116 | 'distinguish_type': 1, | ||
117 | 'auto_rotate': True, | ||
118 | }, | ||
119 | } | ||
120 | for times in range(consts.RETRY_TIMES): | ||
121 | try: | ||
122 | start_time = time.time() | ||
123 | ocr_4_response = requests.post(self.ocr_url_4, json=json_data_4) | ||
124 | if ocr_4_response.status_code != 200: | ||
125 | raise OCR4Exception('ocr_4 status code: {0}'.format(ocr_4_response.status_code)) | ||
126 | except Exception as e: | ||
127 | self.folder_log.warn( | ||
128 | '{0} [ocr_4 failed] [times={1}] [img_path={2}] [error={3}]'.format( | ||
129 | self.log_base, times, img_path, traceback.format_exc())) | ||
130 | else: | ||
131 | ocr_4_res = ocr_4_response.json() | ||
132 | end_time = time.time() | ||
133 | speed_time = int(end_time - start_time) | ||
134 | |||
135 | if ocr_4_res.get('code') == 0 and ocr_4_res.get('result', {}).get('rtn') == 0: | ||
136 | card_type = ocr_4_res.get('result', {}).get( | ||
137 | 'idcard_distinguish_result', {}).get('result', -1) | ||
138 | |||
139 | self.folder_log.info( | ||
140 | '{0} [ocr_4 success] [img_path={1}] [speed_time={2}]'.format( | ||
141 | self.log_base, img_path, speed_time)) | ||
142 | break | ||
143 | else: | ||
144 | self.folder_log.warn( | ||
145 | '{0} [ocr_4 failed] [img_path={1}]'.format(self.log_base, img_path)) | ||
146 | |||
147 | id_card_dict[consts.IC_TURE_OR_FALSE] = consts.IC_RES_MAPPING.get(card_type) | ||
148 | license_summary.setdefault(classify, []).extend(license_data) | ||
149 | |||
150 | @staticmethod | ||
151 | def parse_img_path(img_path): | ||
152 | # 'page_{0}_img_{1}.{2}'.format(pno, img_index, ext) | ||
153 | img_name, _ = os.path.splitext(os.path.basename(img_path)) | ||
154 | if re.match(r'page_\d+_img_\d+', img_name): | ||
155 | part_list = img_name.split('_') | ||
156 | return img_name, int(part_list[1])+1, int(part_list[3])+1 | ||
157 | else: | ||
158 | return img_name, 1, 1 | ||
159 | |||
160 | @staticmethod | ||
161 | def get_path(name, img_output_dir, wb_output_dir, pdf_output_dir): | ||
162 | time_stamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S') | ||
163 | new_name = '{0}_{1}'.format(time_stamp, name) | ||
164 | img_save_path = os.path.join(img_output_dir, new_name) | ||
165 | pdf_save_path = os.path.join(pdf_output_dir, new_name) | ||
166 | excel_name = '{0}.xlsx'.format(os.path.splitext(new_name)[0]) | ||
167 | excel_path = os.path.join(wb_output_dir, excel_name) | ||
168 | return img_save_path, excel_path, pdf_save_path | ||
169 | |||
170 | def res_process(self, all_res, classify, excel_path): | ||
171 | try: | ||
172 | license_summary = {} | ||
173 | |||
174 | if not all_res: | ||
175 | return | ||
176 | else: | ||
177 | for img_path, ocr_res in all_res.items(): | ||
178 | # img_name, pno, ino = self.parse_img_path(img_path) | ||
179 | # part_idx = 1 | ||
180 | |||
181 | if isinstance(ocr_res, dict): | ||
182 | if ocr_res.get('code') == 1: | ||
183 | data_list = ocr_res.get('data', []) | ||
184 | if isinstance(data_list, list): | ||
185 | for ocr_data in data_list: | ||
186 | # part_idx = part_idx + 1 | ||
187 | self.license1_process(ocr_data, license_summary, classify, img_path) | ||
188 | |||
189 | wb = BSWorkbook(set(), set(), set(), set(), set()) | ||
190 | wb.simple_license_rebuild(license_summary, consts.DOC_SCHEME_LIST[0]) | ||
191 | wb.remove_base_sheet() | ||
192 | wb.save(excel_path) | ||
193 | except Exception as e: | ||
194 | self.folder_log.error('{0} [wb build error] [path={1}] [error={2}]'.format( | ||
195 | self.log_base, excel_path, traceback.format_exc())) | ||
196 | |||
197 | def ocr_process(self, img_path, classify): | ||
198 | if os.path.exists(img_path): | ||
199 | # TODO 图片验证 | ||
200 | with open(img_path, 'rb') as f: | ||
201 | base64_data = base64.b64encode(f.read()) | ||
202 | # 获取解码后的base64值 | ||
203 | file_data = base64_data.decode() | ||
204 | json_data = { | ||
205 | "file": file_data, | ||
206 | "classify": classify | ||
207 | } | ||
208 | |||
209 | for times in range(consts.RETRY_TIMES): | ||
210 | try: | ||
211 | start_time = time.time() | ||
212 | ocr_response = requests.post(self.ocr_url, json=json_data) | ||
213 | if ocr_response.status_code != 200: | ||
214 | raise OCR1Exception('{0} ocr status code: {1}'.format(self.log_base, ocr_response.status_code)) | ||
215 | except Exception as e: | ||
216 | self.folder_log.warn('{0} [ocr failed] [times={1}] [img_path={2}] [error={3}]'.format( | ||
217 | self.log_base, times, img_path, traceback.format_exc())) | ||
218 | else: | ||
219 | ocr_res = ocr_response.json() | ||
220 | end_time = time.time() | ||
221 | speed_time = int(end_time - start_time) | ||
222 | self.folder_log.info('{0} [ocr success] [img={1}] [res={2}] [speed_time={3}]'.format( | ||
223 | self.log_base, img_path, ocr_res, speed_time)) | ||
224 | return ocr_res | ||
225 | else: | ||
226 | self.folder_log.warn('{0} [ocr failed] [img_path={1}]'.format(self.log_base, img_path)) | ||
227 | |||
228 | def ltgt_ocr_process(self, img_path_list, label, path): | ||
229 | img_data_list = [] | ||
230 | |||
231 | for img_path in img_path_list: | ||
232 | if os.path.exists(img_path): | ||
233 | with open(img_path, 'rb') as f: | ||
234 | base64_data = base64.b64encode(f.read()) | ||
235 | # 获取解码后的base64值 | ||
236 | file_data = base64_data.decode() | ||
237 | img_data_list.append(file_data) | ||
238 | |||
239 | json_data = { | ||
240 | "label": label, | ||
241 | "img_data_list": img_data_list | ||
242 | } | ||
243 | |||
244 | for times in range(consts.RETRY_TIMES): | ||
245 | try: | ||
246 | start_time = time.time() | ||
247 | ocr_response = requests.post(self.ltgt_ocr_url, json=json_data) | ||
248 | if ocr_response.status_code != 200: | ||
249 | raise LTGTException('{0} ltgt ocr status code: {1}'.format(self.log_base, ocr_response.status_code)) | ||
250 | except Exception as e: | ||
251 | self.folder_log.warn('{0} [ltgt ocr failed] [times={1}] [path={2}] [error={3}]'.format( | ||
252 | self.log_base, times, path, traceback.format_exc())) | ||
253 | else: | ||
254 | ocr_res = ocr_response.json() | ||
255 | end_time = time.time() | ||
256 | speed_time = int(end_time - start_time) | ||
257 | self.folder_log.info('{0} [ltgt ocr success] [path={1}] [res={2}] [speed_time={3}]'.format( | ||
258 | self.log_base, path, ocr_res, speed_time)) | ||
259 | return ocr_res | ||
260 | else: | ||
261 | self.folder_log.warn('{0} [ltgt ocr failed] [path={1}]'.format(self.log_base, path)) | ||
262 | |||
263 | def ltgt_res_process(self, ocr_res, label, excel_path): | ||
264 | try: | ||
265 | if isinstance(ocr_res, dict): | ||
266 | if ocr_res.get('code') == 1: | ||
267 | result_dict = ocr_res.get('data', {}) | ||
268 | |||
269 | wb = BSWorkbook(set(), set(), set(), set(), set()) | ||
270 | rebuild_res = wb.ltgt_build(label, result_dict) | ||
271 | wb.remove_base_sheet() | ||
272 | wb.save(excel_path) | ||
273 | return rebuild_res | ||
274 | except Exception as e: | ||
275 | self.folder_log.error('{0} [wb build error] [path={1}] [error={2}]'.format( | ||
276 | self.log_base, excel_path, traceback.format_exc())) | ||
277 | |||
278 | def ltgt_process(self, img_path_list, label, excel_path, path): | ||
279 | ocr_res = self.ltgt_ocr_process(img_path_list, label, path) | ||
280 | rebuild_res = self.ltgt_res_process(ocr_res, label, excel_path) | ||
281 | return rebuild_res | ||
282 | |||
283 | def images_process(self, img_path_list, classify, excel_path): | ||
284 | all_res = {} | ||
285 | for img_path in img_path_list: | ||
286 | ocr_res = self.ocr_process(img_path, classify) | ||
287 | all_res[img_path] = ocr_res | ||
288 | self.res_process(all_res, classify, excel_path) | ||
289 | |||
290 | def pdf_process(self, name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir): | ||
291 | if os.path.exists(path): | ||
292 | rebuild_res = None | ||
293 | try: | ||
294 | img_save_path, excel_path, pdf_save_path = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir) | ||
295 | self.folder_log.info('{0} [pdf to img start] [path={1}]'.format(self.log_base, path)) | ||
296 | pdf_handler = PDFHandler(path, img_save_path) | ||
297 | if classify in self.ltgt_classify_mapping: | ||
298 | pdf_handler.extract_page_image() | ||
299 | else: | ||
300 | pdf_handler.extract_image() | ||
301 | self.folder_log.info('{0} [pdf to img end] [path={1}]'.format(self.log_base, path)) | ||
302 | except Exception as e: | ||
303 | self.folder_log.error('{0} [pdf to img error] [path={1}] [error={2}]'.format( | ||
304 | self.log_base, path, traceback.format_exc())) | ||
305 | raise e | ||
306 | else: | ||
307 | if classify in self.ltgt_classify_mapping: | ||
308 | rebuild_res = self.ltgt_process(pdf_handler.img_path_list, self.ltgt_classify_mapping[classify], | ||
309 | excel_path, path) | ||
310 | else: | ||
311 | self.images_process(pdf_handler.img_path_list, classify, excel_path) | ||
312 | shutil.move(path, pdf_save_path) | ||
313 | return rebuild_res | ||
314 | |||
315 | def tif_process(self, name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir): | ||
316 | if os.path.exists(path): | ||
317 | rebuild_res = None | ||
318 | try: | ||
319 | img_save_path, excel_path, tiff_save_path = self.get_path(name, img_output_dir, wb_output_dir, tiff_output_dir) | ||
320 | self.folder_log.info('{0} [tiff to img start] [path={1}]'.format(self.log_base, path)) | ||
321 | tiff_handler = TIFFHandler(path, img_save_path) | ||
322 | tiff_handler.extract_image() | ||
323 | self.folder_log.info('{0} [tiff to img end] [path={1}]'.format(self.log_base, path)) | ||
324 | except Exception as e: | ||
325 | self.folder_log.error('{0} [tiff to img error] [path={1}] [error={2}]'.format( | ||
326 | self.log_base, path, traceback.format_exc())) | ||
327 | raise e | ||
328 | else: | ||
329 | if classify in self.ltgt_classify_mapping: | ||
330 | rebuild_res = self.ltgt_process(tiff_handler.img_path_list, self.ltgt_classify_mapping[classify], | ||
331 | excel_path, path) | ||
332 | else: | ||
333 | self.images_process(tiff_handler.img_path_list, classify, excel_path) | ||
334 | shutil.move(path, tiff_save_path) | ||
335 | return rebuild_res | ||
336 | |||
337 | def img_process(self, name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir): | ||
338 | rebuild_res = None | ||
339 | try: | ||
340 | img_save_path, excel_path, _ = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir) | ||
341 | except Exception as e: | ||
342 | self.folder_log.error('{0} [get path error] [path={1}] [error={2}]'.format( | ||
343 | self.log_base, path, traceback.format_exc())) | ||
344 | else: | ||
345 | if classify in self.ltgt_classify_mapping: | ||
346 | rebuild_res = self.ltgt_process([path], self.ltgt_classify_mapping[classify], excel_path, path) | ||
347 | else: | ||
348 | ocr_res = self.ocr_process(path, classify) | ||
349 | all_res = {path: ocr_res} | ||
350 | self.res_process(all_res, classify, excel_path) | ||
351 | shutil.move(path, img_save_path) | ||
352 | return rebuild_res | ||
353 | |||
354 | def wb_process(self, wb_dir, result_queue): | ||
355 | while self.switch: | ||
356 | result_list = [] | ||
357 | date_str = None | ||
358 | for i in range(100): | ||
359 | try: | ||
360 | result = result_queue.get(block=False) | ||
361 | except Exception as e: | ||
362 | time.sleep(self.short_sleep_time) | ||
363 | else: | ||
364 | if date_str is None: | ||
365 | date_str = result[self.DATE_KEY] | ||
366 | result_list.append(result) | ||
367 | elif result[self.DATE_KEY] == date_str: | ||
368 | result_list.append(result) | ||
369 | else: | ||
370 | break | ||
371 | if date_str is None: | ||
372 | time.sleep(self.long_sleep_time) | ||
373 | continue | ||
374 | else: | ||
375 | wb_name = self.daily_wb_name.format(date_str) | ||
376 | wb_path = os.path.join(wb_dir, wb_name) | ||
377 | if os.path.isfile(wb_path): | ||
378 | wb = load_workbook(wb_path) | ||
379 | else: | ||
380 | wb = Workbook() | ||
381 | for result in result_list: | ||
382 | try: | ||
383 | sheet_name, head_fields = self.sheet_content[result[self.CLASSIFY_KEY]] | ||
384 | row = [] | ||
385 | for field in head_fields: | ||
386 | row.append(result[self.RESULT_KEY].get(field)) | ||
387 | if sheet_name in wb.sheetnames: | ||
388 | ws = wb.get_sheet_by_name(sheet_name) | ||
389 | else: | ||
390 | ws = wb.create_sheet(sheet_name) | ||
391 | ws.append(head_fields) | ||
392 | ws.append(row) | ||
393 | except Exception as e: | ||
394 | self.folder_log.info('{0} [daily wb failed] [result={1}] [error={2}]'.format( | ||
395 | self.log_base, result, traceback.format_exc())) | ||
396 | wb.save(wb_path) | ||
397 | |||
398 | def folder_process(self, input_dir, classify, result_queue): | ||
399 | while not os.path.isdir(input_dir): | ||
400 | self.folder_log.info('{0} [input dir is not dir] [input_dir={1}]'.format(self.log_base, input_dir)) | ||
401 | if self.switch: | ||
402 | time.sleep(self.sleep_time) | ||
403 | continue | ||
404 | else: | ||
405 | return | ||
406 | output_dir = os.path.join(os.path.dirname(input_dir), 'Output') | ||
407 | img_output_dir = os.path.join(output_dir, 'image') | ||
408 | wb_output_dir = os.path.join(output_dir, 'excel') | ||
409 | pdf_output_dir = os.path.join(output_dir, 'pdf') | ||
410 | tiff_output_dir = os.path.join(output_dir, 'tiff') | ||
411 | failed_output_dir = os.path.join(output_dir, 'failed') | ||
412 | os.makedirs(output_dir, exist_ok=True) | ||
413 | os.makedirs(img_output_dir, exist_ok=True) | ||
414 | os.makedirs(wb_output_dir, exist_ok=True) | ||
415 | os.makedirs(pdf_output_dir, exist_ok=True) | ||
416 | os.makedirs(tiff_output_dir, exist_ok=True) | ||
417 | os.makedirs(failed_output_dir, exist_ok=True) | ||
418 | os_error_filename_set = set() | ||
419 | while self.switch: | ||
420 | # if not os.path.isdir(input_dir): | ||
421 | # self.folder_log.info('{0} [input dir is not dir] [input_dir={1}]'.format(self.log_base, input_dir)) | ||
422 | # time.sleep(self.sleep_time) | ||
423 | # continue | ||
424 | # 1. 从input dir获取pdf or image | ||
425 | list_dir = os.listdir(input_dir) | ||
426 | if not list_dir and len(os_error_filename_set) == 0: | ||
427 | self.folder_log.info('{0} [input dir empty] [input_dir={1}]'.format(self.log_base, input_dir)) | ||
428 | time.sleep(self.sleep_time) | ||
429 | continue | ||
430 | all_file_set = set(list_dir) | ||
431 | true_file_set = all_file_set - os_error_filename_set | ||
432 | if len(true_file_set) == 0 and len(os_error_filename_set) > 0: | ||
433 | true_file_set.add(os_error_filename_set.pop()) | ||
434 | for name in true_file_set: | ||
435 | path = os.path.join(input_dir, name) | ||
436 | |||
437 | try: | ||
438 | if os.path.isfile(path): | ||
439 | self.folder_log.info('{0} [file start] [path={1}]'.format(self.log_base, path)) | ||
440 | if name.endswith('.pdf') or name.endswith('.PDF'): | ||
441 | result = self.pdf_process(name, path, classify, img_output_dir, | ||
442 | wb_output_dir, pdf_output_dir) | ||
443 | elif name.endswith('.tif') or name.endswith('.TIF'): | ||
444 | result = self.tif_process(name, path, classify, img_output_dir, | ||
445 | wb_output_dir, tiff_output_dir) | ||
446 | else: | ||
447 | result = self.img_process(name, path, classify, wb_output_dir, | ||
448 | img_output_dir, pdf_output_dir) | ||
449 | self.folder_log.info('{0} [file end] [path={1}]'.format(self.log_base, path)) | ||
450 | else: | ||
451 | result = None | ||
452 | self.folder_log.info('{0} [path is dir] [path={1}]'.format(self.log_base, input_dir)) | ||
453 | failed_path = os.path.join(failed_output_dir, '{0}_{1}'.format(time.time(), name)) | ||
454 | shutil.move(path, failed_path) | ||
455 | except OSError: | ||
456 | os_error_filename_set.add(name) | ||
457 | self.folder_log.error('{0} [os error] [path={1}] [error={2}]'.format( | ||
458 | self.log_base, path, traceback.format_exc())) | ||
459 | except Exception as e: | ||
460 | try: | ||
461 | self.folder_log.error('{0} [file error] [path={1}] [error={2}]'.format(self.log_base, path, | ||
462 | traceback.format_exc())) | ||
463 | failed_path = os.path.join(failed_output_dir, '{0}_{1}'.format(time.time(), name)) | ||
464 | shutil.move(path, failed_path) | ||
465 | except Exception as e: | ||
466 | os_error_filename_set.add(name) | ||
467 | self.folder_log.error('{0} [file move error] [path={1}] [error={2}]'.format( | ||
468 | self.log_base, path, traceback.format_exc())) | ||
469 | else: | ||
470 | if isinstance(result, dict) and len(result) > 0: | ||
471 | date_str = time.strftime("%Y-%m-%d") | ||
472 | result_queue.put( | ||
473 | { | ||
474 | self.CLASSIFY_KEY: classify, | ||
475 | self.RESULT_KEY: result, | ||
476 | self.DATE_KEY: date_str | ||
477 | } | ||
478 | ) | ||
479 | elif isinstance(result, list) and len(result) > 0: | ||
480 | date_str = time.strftime("%Y-%m-%d") | ||
481 | for res in result: | ||
482 | result_queue.put( | ||
483 | { | ||
484 | self.CLASSIFY_KEY: classify, | ||
485 | self.RESULT_KEY: res, | ||
486 | self.DATE_KEY: date_str | ||
487 | } | ||
488 | ) | ||
489 | |||
490 | def handle(self, *args, **kwargs): | ||
491 | if len(self.input_dirs) == 0: | ||
492 | return | ||
493 | result_queue = Queue() | ||
494 | process_list = [] | ||
495 | one_input_dir = None | ||
496 | for classify_idx, input_dir in self.input_dirs.items(): | ||
497 | if one_input_dir is None: | ||
498 | one_input_dir = input_dir | ||
499 | classify = int(classify_idx.split('_')[0]) | ||
500 | process = Process(target=self.folder_process, args=(input_dir, classify, result_queue)) | ||
501 | process_list.append(process) | ||
502 | |||
503 | wb_dir = os.path.dirname(os.path.dirname(one_input_dir)) | ||
504 | wb_process = Process(target=self.wb_process, args=(wb_dir, result_queue, )) | ||
505 | process_list.append(wb_process) | ||
506 | |||
507 | for p in process_list: | ||
508 | p.start() | ||
509 | for p in process_list: | ||
510 | p.join() | ||
511 | |||
512 | self.folder_log.info('{0} [stop safely]'.format(self.log_base)) |
... | @@ -61,13 +61,11 @@ class Command(BaseCommand, LoggerMixin): | ... | @@ -61,13 +61,11 @@ class Command(BaseCommand, LoggerMixin): |
61 | def signal_handler(self, sig, frame): | 61 | def signal_handler(self, sig, frame): |
62 | self.switch = False # 停止处理文件 | 62 | self.switch = False # 停止处理文件 |
63 | 63 | ||
64 | def license1_process(self, ocr_data, license_summary, classify, res_list, pno, ino, part_idx, img_path): | 64 | def license1_process(self, ocr_data, license_summary, classify, img_path): |
65 | # 类别:'0'身份证, '1'居住证 | 65 | # 类别:'0'身份证, '1'居住证 |
66 | license_data = ocr_data.get('data', []) | 66 | license_data = ocr_data.get('data', []) |
67 | if not license_data: | 67 | if not license_data: |
68 | res_list.append((pno, ino, part_idx, consts.RES_SUCCESS_EMPTY)) | ||
69 | return | 68 | return |
70 | res_list.append((pno, ino, part_idx, consts.RES_SUCCESS)) | ||
71 | if classify == consts.MVC_CLASSIFY: # 车辆登记证 3/4页结果整合 | 69 | if classify == consts.MVC_CLASSIFY: # 车辆登记证 3/4页结果整合 |
72 | for mvc_dict in license_data: | 70 | for mvc_dict in license_data: |
73 | try: | 71 | try: |
... | @@ -154,29 +152,21 @@ class Command(BaseCommand, LoggerMixin): | ... | @@ -154,29 +152,21 @@ class Command(BaseCommand, LoggerMixin): |
154 | def res_process(self, all_res, classify, excel_path): | 152 | def res_process(self, all_res, classify, excel_path): |
155 | try: | 153 | try: |
156 | license_summary = {} | 154 | license_summary = {} |
157 | res_list = [] | ||
158 | 155 | ||
159 | if not all_res: | 156 | if not all_res: |
160 | return | 157 | return |
161 | else: | 158 | else: |
162 | for img_path, ocr_res in all_res.items(): | 159 | for img_path, ocr_res in all_res.items(): |
163 | img_name, pno, ino = self.parse_img_path(img_path) | 160 | # img_name, pno, ino = self.parse_img_path(img_path) |
164 | part_idx = 1 | 161 | # part_idx = 1 |
165 | 162 | ||
166 | if isinstance(ocr_res, dict): | 163 | if isinstance(ocr_res, dict): |
167 | if ocr_res.get('code') == 1: | 164 | if ocr_res.get('code') == 1: |
168 | data_list = ocr_res.get('data', []) | 165 | data_list = ocr_res.get('data', []) |
169 | if isinstance(data_list, list): | 166 | if isinstance(data_list, list): |
170 | for part_idx, ocr_data in enumerate(data_list): | 167 | for ocr_data in data_list: |
171 | part_idx = part_idx + 1 | 168 | # part_idx = part_idx + 1 |
172 | self.license1_process(ocr_data, license_summary, classify, | 169 | self.license1_process(ocr_data, license_summary, classify, img_path) |
173 | res_list, pno, ino, part_idx, img_path) | ||
174 | else: | ||
175 | res_list.append((pno, ino, part_idx, consts.RES_FAILED_3)) | ||
176 | else: | ||
177 | res_list.append((pno, ino, part_idx, consts.RES_FAILED)) | ||
178 | else: | ||
179 | res_list.append((pno, ino, part_idx, consts.RES_FAILED)) | ||
180 | 170 | ||
181 | wb = BSWorkbook(set(), set(), set(), set(), set()) | 171 | wb = BSWorkbook(set(), set(), set(), set(), set()) |
182 | wb.simple_license_rebuild(license_summary, consts.DOC_SCHEME_LIST[0]) | 172 | wb.simple_license_rebuild(license_summary, consts.DOC_SCHEME_LIST[0]) |
... | @@ -217,6 +207,13 @@ class Command(BaseCommand, LoggerMixin): | ... | @@ -217,6 +207,13 @@ class Command(BaseCommand, LoggerMixin): |
217 | else: | 207 | else: |
218 | self.folder_log.warn('{0} [ocr failed] [img_path={1}]'.format(self.log_base, img_path)) | 208 | self.folder_log.warn('{0} [ocr failed] [img_path={1}]'.format(self.log_base, img_path)) |
219 | 209 | ||
210 | def images_process(self, img_path_list, classify, excel_path): | ||
211 | all_res = {} | ||
212 | for img_path in img_path_list: | ||
213 | ocr_res = self.ocr_process(img_path, classify) | ||
214 | all_res[img_path] = ocr_res | ||
215 | self.res_process(all_res, classify, excel_path) | ||
216 | |||
220 | def pdf_process(self, name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir): | 217 | def pdf_process(self, name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir): |
221 | if os.path.exists(path): | 218 | if os.path.exists(path): |
222 | try: | 219 | try: |
... | @@ -230,11 +227,7 @@ class Command(BaseCommand, LoggerMixin): | ... | @@ -230,11 +227,7 @@ class Command(BaseCommand, LoggerMixin): |
230 | self.log_base, path, traceback.format_exc())) | 227 | self.log_base, path, traceback.format_exc())) |
231 | raise e | 228 | raise e |
232 | else: | 229 | else: |
233 | all_res = {} | 230 | self.images_process(pdf_handler.img_path_list, classify, excel_path) |
234 | for img_path in pdf_handler.img_path_list: | ||
235 | ocr_res = self.ocr_process(img_path, classify) | ||
236 | all_res[img_path] = ocr_res | ||
237 | self.res_process(all_res, classify, excel_path) | ||
238 | shutil.move(path, pdf_save_path) | 231 | shutil.move(path, pdf_save_path) |
239 | 232 | ||
240 | def tif_process(self, name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir): | 233 | def tif_process(self, name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir): |
... | @@ -250,23 +243,18 @@ class Command(BaseCommand, LoggerMixin): | ... | @@ -250,23 +243,18 @@ class Command(BaseCommand, LoggerMixin): |
250 | self.log_base, path, traceback.format_exc())) | 243 | self.log_base, path, traceback.format_exc())) |
251 | raise e | 244 | raise e |
252 | else: | 245 | else: |
253 | all_res = {} | 246 | self.images_process(tiff_handler.img_path_list, classify, excel_path) |
254 | for img_path in tiff_handler.img_path_list: | ||
255 | ocr_res = self.ocr_process(img_path, classify) | ||
256 | all_res[img_path] = ocr_res | ||
257 | self.res_process(all_res, classify, excel_path) | ||
258 | shutil.move(path, tiff_save_path) | 247 | shutil.move(path, tiff_save_path) |
259 | 248 | ||
260 | def img_process(self, name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir): | 249 | def img_process(self, name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir): |
261 | ocr_res = self.ocr_process(path, classify) | ||
262 | all_res = {path: ocr_res} | ||
263 | |||
264 | try: | 250 | try: |
265 | img_save_path, excel_path, _ = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir) | 251 | img_save_path, excel_path, _ = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir) |
266 | except Exception as e: | 252 | except Exception as e: |
267 | self.folder_log.error('{0} [get path error] [path={1}] [error={2}]'.format( | 253 | self.folder_log.error('{0} [get path error] [path={1}] [error={2}]'.format( |
268 | self.log_base, path, traceback.format_exc())) | 254 | self.log_base, path, traceback.format_exc())) |
269 | else: | 255 | else: |
256 | ocr_res = self.ocr_process(path, classify) | ||
257 | all_res = {path: ocr_res} | ||
270 | self.res_process(all_res, classify, excel_path) | 258 | self.res_process(all_res, classify, excel_path) |
271 | shutil.move(path, img_save_path) | 259 | shutil.move(path, img_save_path) |
272 | 260 | ||
... | @@ -312,9 +300,9 @@ class Command(BaseCommand, LoggerMixin): | ... | @@ -312,9 +300,9 @@ class Command(BaseCommand, LoggerMixin): |
312 | try: | 300 | try: |
313 | if os.path.isfile(path): | 301 | if os.path.isfile(path): |
314 | self.folder_log.info('{0} [file start] [path={1}]'.format(self.log_base, path)) | 302 | self.folder_log.info('{0} [file start] [path={1}]'.format(self.log_base, path)) |
315 | if name.endswith('.pdf'): | 303 | if name.endswith('.pdf') or name.endswith('.PDF'): |
316 | self.pdf_process(name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir) | 304 | self.pdf_process(name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir) |
317 | elif name.endswith('.tif'): | 305 | elif name.endswith('.tif') or name.endswith('.TIF'): |
318 | self.tif_process(name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir) | 306 | self.tif_process(name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir) |
319 | else: | 307 | else: |
320 | self.img_process(name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir) | 308 | self.img_process(name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir) | ... | ... |
... | @@ -702,6 +702,31 @@ class BSWorkbook(Workbook): | ... | @@ -702,6 +702,31 @@ class BSWorkbook(Workbook): |
702 | if field_str is not None: | 702 | if field_str is not None: |
703 | count_list.append((field_str, count)) | 703 | count_list.append((field_str, count)) |
704 | 704 | ||
705 | def ltgt_build(self, label, result_dict): | ||
706 | ws = self.create_sheet(label) | ||
707 | rebuild_res = {} | ||
708 | for key, value in result_dict.items(): | ||
709 | if isinstance(value, list): | ||
710 | value_list = [dict_item.get('words') for dict_item in value] | ||
711 | ws.append((key, '、'.join(value_list))) | ||
712 | rebuild_res[key] = '、'.join(value_list) | ||
713 | elif isinstance(value, dict): | ||
714 | if 'words' in value: | ||
715 | ws.append((key, value['words'])) | ||
716 | rebuild_res[key] = value['words'] | ||
717 | else: | ||
718 | for sub_key, sub_value in value.items(): | ||
719 | if isinstance(sub_value, dict): | ||
720 | ws.append(('{0}: {1}'.format(key, sub_key), sub_value.get('words', ''))) | ||
721 | rebuild_res['{0}: {1}'.format(key, sub_key)] = sub_value.get('words', '') | ||
722 | else: | ||
723 | ws.append(('{0}: {1}'.format(key, sub_key), sub_value)) | ||
724 | rebuild_res['{0}: {1}'.format(key, sub_key)] = sub_value | ||
725 | else: | ||
726 | ws.append((key, value)) | ||
727 | rebuild_res[key] = value | ||
728 | return rebuild_res | ||
729 | |||
705 | def simple_license_rebuild(self, license_summary, document_scheme): | 730 | def simple_license_rebuild(self, license_summary, document_scheme): |
706 | # for ic_license_dict in license_summary.get(consts.IC_CLASSIFY, []): | 731 | # for ic_license_dict in license_summary.get(consts.IC_CLASSIFY, []): |
707 | # if ic_license_dict.get('类别') == '1': | 732 | # if ic_license_dict.get('类别') == '1': | ... | ... |
... | @@ -225,3 +225,13 @@ class PDFHandler: | ... | @@ -225,3 +225,13 @@ class PDFHandler: |
225 | else: | 225 | else: |
226 | self.merge_il(pdf, pno, il) | 226 | self.merge_il(pdf, pno, il) |
227 | self.img_count = len(self.img_path_list) | 227 | self.img_count = len(self.img_path_list) |
228 | |||
229 | def extract_page_image(self): | ||
230 | self.img_path_list = [] | ||
231 | self.xref_set = set() | ||
232 | os.makedirs(self.img_dir_path, exist_ok=True) | ||
233 | with fitz.Document(self.path) as pdf: | ||
234 | for pno in range(pdf.pageCount): | ||
235 | page = pdf.loadPage(pno) | ||
236 | self.page_to_png(page) | ||
237 | self.img_count = len(self.img_path_list) | ... | ... |
-
Please register or sign in to post a comment