Skip to content

周伟奇 / bmw-ocr

Go to a project
Toggle navigation
Toggle navigation pinning
423427c0 by 周伟奇
Options

Merge branch 'feature/ltgt' into feature/0611

2 parents d78669c5 68d7dd98
Inline Side-by-side
Showing 5 changed files with 569 additions and 31 deletions
src/apps/doc/exceptions.py
...@@ -13,6 +13,9 @@ class OCR2Exception(Exception): ...@@ -13,6 +13,9 @@ class OCR2Exception(Exception):
13 class OCR4Exception(Exception): 13 class OCR4Exception(Exception):
14 pass 14 pass
15 15
16 class LTGTException(Exception):
17 pass
18
16 19
17 class GCAPException(Exception): 20 class GCAPException(Exception):
18 pass 21 pass
......
src/apps/doc/management/commands/folder_ltgt_process.py 0 → 100644
1 import os
2 import re
3 import time
4 import shutil
5 import base64
6 import signal
7 import requests
8 import traceback
9 from PIL import Image
10 from datetime import datetime
11 from django.core.management import BaseCommand
12 from multiprocessing import Process, Queue
13 from openpyxl import load_workbook, Workbook
14
15 from settings import conf
16 from common.mixins import LoggerMixin
17 from common.tools.pdf_to_img import PDFHandler
18 from apps.doc import consts
19 from apps.doc.exceptions import OCR1Exception, OCR4Exception, LTGTException
20 from apps.doc.ocr.wb import BSWorkbook
21
22
23 class TIFFHandler:
24
25 def __init__(self, path, img_save_path):
26 self.path = path
27 self.img_save_path = img_save_path
28 self.img_path_list = []
29
30 def extract_image(self):
31 os.makedirs(self.img_save_path, exist_ok=True)
32 tiff = Image.open(self.path)
33 tiff.load()
34
35 for i in range(tiff.n_frames):
36 try:
37 save_path = os.path.join(self.img_save_path, 'page_{0}.jpeg'.format(i))
38 tiff.seek(i)
39 tiff.save(save_path)
40 self.img_path_list.append(save_path)
41 except EOFError:
42 break
43
44
45 class Command(BaseCommand, LoggerMixin):
46
47 def __init__(self):
48 super().__init__()
49 self.log_base = '[folder ltgt process]'
50 # 处理文件开关
51 self.switch = True
52 self.ltgt_classify_mapping = {
53 128: '执行裁定书',
54 129: '民事判决书',
55 130: '民事调解书'
56 }
57 self.sheet_content = {
58 128: ('执行裁定书', ('承办法院', '案号/标号', '被执行人', '债权金额', '诉讼时间')),
59 129: ('民事判决书', ('承办法院', '案号/标号', '被告', '判决结果: 贷款本金', '判决结果: 罚息', '判决结果: 律师费', '判决结果: 案件受理费', '诉讼时间')),
60 130: ('民事调解书', ('承办法院', '案号/标号', '被告', '协议内容: 支付金额', '协议内容: 案件受理费', '诉讼时间')),
61 }
62 self.DATE_KEY = 'date'
63 self.CLASSIFY_KEY = 'classify'
64 self.RESULT_KEY = 'result'
65 self.daily_wb_name = 'Output_{0}.xlsx'
66 self.short_sleep_time = 10
67 self.long_sleep_time = 3600
68 # 睡眠时间
69 self.sleep_time = float(conf.SLEEP_SECOND_FOLDER)
70 # input folder
71 self.input_dirs = conf.get_namespace('LTGT_DIR_')
72 # ocr相关
73 # self.ocr_url = conf.OCR_URL_FOLDER
74 # self.ocr_url_4 = conf.IC_URL
75 self.ltgt_ocr_url = conf.LTGT_URL
76 # 优雅退出信号：15
77 signal.signal(signal.SIGTERM, self.signal_handler)
78
79 def signal_handler(self, sig, frame):
80 self.switch = False # 停止处理文件
81
82 def license1_process(self, ocr_data, license_summary, classify, img_path):
83 # 类别：'0'身份证， '1'居住证
84 license_data = ocr_data.get('data', [])
85 if not license_data:
86 return
87 if classify == consts.MVC_CLASSIFY: # 车辆登记证 3/4页结果整合
88 for mvc_dict in license_data:
89 try:
90 mvc_page = mvc_dict.pop('page')
91 except Exception as e:
92 pass
93 else:
94 if mvc_page == 'VehicleRegArea':
95 mvc_res = mvc_dict.pop('results', {})
96 mvc_dict['机动车登记证书编号'] = mvc_res.get('register_no', {}).get('words', '')
97 for register_info in mvc_res.get('register_info', []):
98 for detail_dict in register_info.get('details', {}).values():
99 mvc_dict.setdefault(detail_dict.get('chinese_key', '未知'), []).append(
100 detail_dict.get('words', ''))
101 del mvc_res
102 if classify == consts.IC_CLASSIFY:
103 for id_card_dict in license_data:
104 try:
105 base64_img = id_card_dict.pop('base64_img')
106 except Exception as e:
107 continue
108 else:
109 card_type = -1
110 json_data_4 = {
111 'mode': 1,
112 'user_info': {
113 'image_content': base64_img,
114 },
115 'options': {
116 'distinguish_type': 1,
117 'auto_rotate': True,
118 },
119 }
120 for times in range(consts.RETRY_TIMES):
121 try:
122 start_time = time.time()
123 ocr_4_response = requests.post(self.ocr_url_4, json=json_data_4)
124 if ocr_4_response.status_code != 200:
125 raise OCR4Exception('ocr_4 status code: {0}'.format(ocr_4_response.status_code))
126 except Exception as e:
127 self.folder_log.warn(
128 '{0} [ocr_4 failed] [times={1}] [img_path={2}] [error={3}]'.format(
129 self.log_base, times, img_path, traceback.format_exc()))
130 else:
131 ocr_4_res = ocr_4_response.json()
132 end_time = time.time()
133 speed_time = int(end_time - start_time)
134
135 if ocr_4_res.get('code') == 0 and ocr_4_res.get('result', {}).get('rtn') == 0:
136 card_type = ocr_4_res.get('result', {}).get(
137 'idcard_distinguish_result', {}).get('result', -1)
138
139 self.folder_log.info(
140 '{0} [ocr_4 success] [img_path={1}] [speed_time={2}]'.format(
141 self.log_base, img_path, speed_time))
142 break
143 else:
144 self.folder_log.warn(
145 '{0} [ocr_4 failed] [img_path={1}]'.format(self.log_base, img_path))
146
147 id_card_dict[consts.IC_TURE_OR_FALSE] = consts.IC_RES_MAPPING.get(card_type)
148 license_summary.setdefault(classify, []).extend(license_data)
149
150 @staticmethod
151 def parse_img_path(img_path):
152 # 'page_{0}_img_{1}.{2}'.format(pno, img_index, ext)
153 img_name, _ = os.path.splitext(os.path.basename(img_path))
154 if re.match(r'page_\d+_img_\d+', img_name):
155 part_list = img_name.split('_')
156 return img_name, int(part_list[1])+1, int(part_list[3])+1
157 else:
158 return img_name, 1, 1
159
160 @staticmethod
161 def get_path(name, img_output_dir, wb_output_dir, pdf_output_dir):
162 time_stamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
163 new_name = '{0}_{1}'.format(time_stamp, name)
164 img_save_path = os.path.join(img_output_dir, new_name)
165 pdf_save_path = os.path.join(pdf_output_dir, new_name)
166 excel_name = '{0}.xlsx'.format(os.path.splitext(new_name)[0])
167 excel_path = os.path.join(wb_output_dir, excel_name)
168 return img_save_path, excel_path, pdf_save_path
169
170 def res_process(self, all_res, classify, excel_path):
171 try:
172 license_summary = {}
173
174 if not all_res:
175 return
176 else:
177 for img_path, ocr_res in all_res.items():
178 # img_name, pno, ino = self.parse_img_path(img_path)
179 # part_idx = 1
180
181 if isinstance(ocr_res, dict):
182 if ocr_res.get('code') == 1:
183 data_list = ocr_res.get('data', [])
184 if isinstance(data_list, list):
185 for ocr_data in data_list:
186 # part_idx = part_idx + 1
187 self.license1_process(ocr_data, license_summary, classify, img_path)
188
189 wb = BSWorkbook(set(), set(), set(), set(), set())
190 wb.simple_license_rebuild(license_summary, consts.DOC_SCHEME_LIST[0])
191 wb.remove_base_sheet()
192 wb.save(excel_path)
193 except Exception as e:
194 self.folder_log.error('{0} [wb build error] [path={1}] [error={2}]'.format(
195 self.log_base, excel_path, traceback.format_exc()))
196
197 def ocr_process(self, img_path, classify):
198 if os.path.exists(img_path):
199 # TODO 图片验证
200 with open(img_path, 'rb') as f:
201 base64_data = base64.b64encode(f.read())
202 # 获取解码后的base64值
203 file_data = base64_data.decode()
204 json_data = {
205 "file": file_data,
206 "classify": classify
207 }
208
209 for times in range(consts.RETRY_TIMES):
210 try:
211 start_time = time.time()
212 ocr_response = requests.post(self.ocr_url, json=json_data)
213 if ocr_response.status_code != 200:
214 raise OCR1Exception('{0} ocr status code: {1}'.format(self.log_base, ocr_response.status_code))
215 except Exception as e:
216 self.folder_log.warn('{0} [ocr failed] [times={1}] [img_path={2}] [error={3}]'.format(
217 self.log_base, times, img_path, traceback.format_exc()))
218 else:
219 ocr_res = ocr_response.json()
220 end_time = time.time()
221 speed_time = int(end_time - start_time)
222 self.folder_log.info('{0} [ocr success] [img={1}] [res={2}] [speed_time={3}]'.format(
223 self.log_base, img_path, ocr_res, speed_time))
224 return ocr_res
225 else:
226 self.folder_log.warn('{0} [ocr failed] [img_path={1}]'.format(self.log_base, img_path))
227
228 def ltgt_ocr_process(self, img_path_list, label, path):
229 img_data_list = []
230
231 for img_path in img_path_list:
232 if os.path.exists(img_path):
233 with open(img_path, 'rb') as f:
234 base64_data = base64.b64encode(f.read())
235 # 获取解码后的base64值
236 file_data = base64_data.decode()
237 img_data_list.append(file_data)
238
239 json_data = {
240 "label": label,
241 "img_data_list": img_data_list
242 }
243
244 for times in range(consts.RETRY_TIMES):
245 try:
246 start_time = time.time()
247 ocr_response = requests.post(self.ltgt_ocr_url, json=json_data)
248 if ocr_response.status_code != 200:
249 raise LTGTException('{0} ltgt ocr status code: {1}'.format(self.log_base, ocr_response.status_code))
250 except Exception as e:
251 self.folder_log.warn('{0} [ltgt ocr failed] [times={1}] [path={2}] [error={3}]'.format(
252 self.log_base, times, path, traceback.format_exc()))
253 else:
254 ocr_res = ocr_response.json()
255 end_time = time.time()
256 speed_time = int(end_time - start_time)
257 self.folder_log.info('{0} [ltgt ocr success] [path={1}] [res={2}] [speed_time={3}]'.format(
258 self.log_base, path, ocr_res, speed_time))
259 return ocr_res
260 else:
261 self.folder_log.warn('{0} [ltgt ocr failed] [path={1}]'.format(self.log_base, path))
262
263 def ltgt_res_process(self, ocr_res, label, excel_path):
264 try:
265 if isinstance(ocr_res, dict):
266 if ocr_res.get('code') == 1:
267 result_dict = ocr_res.get('data', {})
268
269 wb = BSWorkbook(set(), set(), set(), set(), set())
270 rebuild_res = wb.ltgt_build(label, result_dict)
271 wb.remove_base_sheet()
272 wb.save(excel_path)
273 return rebuild_res
274 except Exception as e:
275 self.folder_log.error('{0} [wb build error] [path={1}] [error={2}]'.format(
276 self.log_base, excel_path, traceback.format_exc()))
277
278 def ltgt_process(self, img_path_list, label, excel_path, path):
279 ocr_res = self.ltgt_ocr_process(img_path_list, label, path)
280 rebuild_res = self.ltgt_res_process(ocr_res, label, excel_path)
281 return rebuild_res
282
283 def images_process(self, img_path_list, classify, excel_path):
284 all_res = {}
285 for img_path in img_path_list:
286 ocr_res = self.ocr_process(img_path, classify)
287 all_res[img_path] = ocr_res
288 self.res_process(all_res, classify, excel_path)
289
290 def pdf_process(self, name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir):
291 if os.path.exists(path):
292 rebuild_res = None
293 try:
294 img_save_path, excel_path, pdf_save_path = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir)
295 self.folder_log.info('{0} [pdf to img start] [path={1}]'.format(self.log_base, path))
296 pdf_handler = PDFHandler(path, img_save_path)
297 if classify in self.ltgt_classify_mapping:
298 pdf_handler.extract_page_image()
299 else:
300 pdf_handler.extract_image()
301 self.folder_log.info('{0} [pdf to img end] [path={1}]'.format(self.log_base, path))
302 except Exception as e:
303 self.folder_log.error('{0} [pdf to img error] [path={1}] [error={2}]'.format(
304 self.log_base, path, traceback.format_exc()))
305 raise e
306 else:
307 if classify in self.ltgt_classify_mapping:
308 rebuild_res = self.ltgt_process(pdf_handler.img_path_list, self.ltgt_classify_mapping[classify],
309 excel_path, path)
310 else:
311 self.images_process(pdf_handler.img_path_list, classify, excel_path)
312 shutil.move(path, pdf_save_path)
313 return rebuild_res
314
315 def tif_process(self, name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir):
316 if os.path.exists(path):
317 rebuild_res = None
318 try:
319 img_save_path, excel_path, tiff_save_path = self.get_path(name, img_output_dir, wb_output_dir, tiff_output_dir)
320 self.folder_log.info('{0} [tiff to img start] [path={1}]'.format(self.log_base, path))
321 tiff_handler = TIFFHandler(path, img_save_path)
322 tiff_handler.extract_image()
323 self.folder_log.info('{0} [tiff to img end] [path={1}]'.format(self.log_base, path))
324 except Exception as e:
325 self.folder_log.error('{0} [tiff to img error] [path={1}] [error={2}]'.format(
326 self.log_base, path, traceback.format_exc()))
327 raise e
328 else:
329 if classify in self.ltgt_classify_mapping:
330 rebuild_res = self.ltgt_process(tiff_handler.img_path_list, self.ltgt_classify_mapping[classify],
331 excel_path, path)
332 else:
333 self.images_process(tiff_handler.img_path_list, classify, excel_path)
334 shutil.move(path, tiff_save_path)
335 return rebuild_res
336
337 def img_process(self, name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir):
338 rebuild_res = None
339 try:
340 img_save_path, excel_path, _ = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir)
341 except Exception as e:
342 self.folder_log.error('{0} [get path error] [path={1}] [error={2}]'.format(
343 self.log_base, path, traceback.format_exc()))
344 else:
345 if classify in self.ltgt_classify_mapping:
346 rebuild_res = self.ltgt_process([path], self.ltgt_classify_mapping[classify], excel_path, path)
347 else:
348 ocr_res = self.ocr_process(path, classify)
349 all_res = {path: ocr_res}
350 self.res_process(all_res, classify, excel_path)
351 shutil.move(path, img_save_path)
352 return rebuild_res
353
354 def wb_process(self, wb_dir, result_queue):
355 while self.switch:
356 result_list = []
357 date_str = None
358 for i in range(100):
359 try:
360 result = result_queue.get(block=False)
361 except Exception as e:
362 time.sleep(self.short_sleep_time)
363 else:
364 if date_str is None:
365 date_str = result[self.DATE_KEY]
366 result_list.append(result)
367 elif result[self.DATE_KEY] == date_str:
368 result_list.append(result)
369 else:
370 break
371 if date_str is None:
372 time.sleep(self.long_sleep_time)
373 continue
374 else:
375 wb_name = self.daily_wb_name.format(date_str)
376 wb_path = os.path.join(wb_dir, wb_name)
377 if os.path.isfile(wb_path):
378 wb = load_workbook(wb_path)
379 else:
380 wb = Workbook()
381 for result in result_list:
382 try:
383 sheet_name, head_fields = self.sheet_content[result[self.CLASSIFY_KEY]]
384 row = []
385 for field in head_fields:
386 row.append(result[self.RESULT_KEY].get(field))
387 if sheet_name in wb.sheetnames:
388 ws = wb.get_sheet_by_name(sheet_name)
389 else:
390 ws = wb.create_sheet(sheet_name)
391 ws.append(head_fields)
392 ws.append(row)
393 except Exception as e:
394 self.folder_log.info('{0} [daily wb failed] [result={1}] [error={2}]'.format(
395 self.log_base, result, traceback.format_exc()))
396 wb.save(wb_path)
397
398 def folder_process(self, input_dir, classify, result_queue):
399 while not os.path.isdir(input_dir):
400 self.folder_log.info('{0} [input dir is not dir] [input_dir={1}]'.format(self.log_base, input_dir))
401 if self.switch:
402 time.sleep(self.sleep_time)
403 continue
404 else:
405 return
406 output_dir = os.path.join(os.path.dirname(input_dir), 'Output')
407 img_output_dir = os.path.join(output_dir, 'image')
408 wb_output_dir = os.path.join(output_dir, 'excel')
409 pdf_output_dir = os.path.join(output_dir, 'pdf')
410 tiff_output_dir = os.path.join(output_dir, 'tiff')
411 failed_output_dir = os.path.join(output_dir, 'failed')
412 os.makedirs(output_dir, exist_ok=True)
413 os.makedirs(img_output_dir, exist_ok=True)
414 os.makedirs(wb_output_dir, exist_ok=True)
415 os.makedirs(pdf_output_dir, exist_ok=True)
416 os.makedirs(tiff_output_dir, exist_ok=True)
417 os.makedirs(failed_output_dir, exist_ok=True)
418 os_error_filename_set = set()
419 while self.switch:
420 # if not os.path.isdir(input_dir):
421 # self.folder_log.info('{0} [input dir is not dir] [input_dir={1}]'.format(self.log_base, input_dir))
422 # time.sleep(self.sleep_time)
423 # continue
424 # 1. 从input dir获取pdf or image
425 list_dir = os.listdir(input_dir)
426 if not list_dir and len(os_error_filename_set) == 0:
427 self.folder_log.info('{0} [input dir empty] [input_dir={1}]'.format(self.log_base, input_dir))
428 time.sleep(self.sleep_time)
429 continue
430 all_file_set = set(list_dir)
431 true_file_set = all_file_set - os_error_filename_set
432 if len(true_file_set) == 0 and len(os_error_filename_set) > 0:
433 true_file_set.add(os_error_filename_set.pop())
434 for name in true_file_set:
435 path = os.path.join(input_dir, name)
436
437 try:
438 if os.path.isfile(path):
439 self.folder_log.info('{0} [file start] [path={1}]'.format(self.log_base, path))
440 if name.endswith('.pdf') or name.endswith('.PDF'):
441 result = self.pdf_process(name, path, classify, img_output_dir,
442 wb_output_dir, pdf_output_dir)
443 elif name.endswith('.tif') or name.endswith('.TIF'):
444 result = self.tif_process(name, path, classify, img_output_dir,
445 wb_output_dir, tiff_output_dir)
446 else:
447 result = self.img_process(name, path, classify, wb_output_dir,
448 img_output_dir, pdf_output_dir)
449 self.folder_log.info('{0} [file end] [path={1}]'.format(self.log_base, path))
450 else:
451 result = None
452 self.folder_log.info('{0} [path is dir] [path={1}]'.format(self.log_base, input_dir))
453 failed_path = os.path.join(failed_output_dir, '{0}_{1}'.format(time.time(), name))
454 shutil.move(path, failed_path)
455 except OSError:
456 os_error_filename_set.add(name)
457 self.folder_log.error('{0} [os error] [path={1}] [error={2}]'.format(
458 self.log_base, path, traceback.format_exc()))
459 except Exception as e:
460 try:
461 self.folder_log.error('{0} [file error] [path={1}] [error={2}]'.format(self.log_base, path,
462 traceback.format_exc()))
463 failed_path = os.path.join(failed_output_dir, '{0}_{1}'.format(time.time(), name))
464 shutil.move(path, failed_path)
465 except Exception as e:
466 os_error_filename_set.add(name)
467 self.folder_log.error('{0} [file move error] [path={1}] [error={2}]'.format(
468 self.log_base, path, traceback.format_exc()))
469 else:
470 if isinstance(result, dict) and len(result) > 0:
471 date_str = time.strftime("%Y-%m-%d")
472 result_queue.put(
473 {
474 self.CLASSIFY_KEY: classify,
475 self.RESULT_KEY: result,
476 self.DATE_KEY: date_str
477 }
478 )
479 elif isinstance(result, list) and len(result) > 0:
480 date_str = time.strftime("%Y-%m-%d")
481 for res in result:
482 result_queue.put(
483 {
484 self.CLASSIFY_KEY: classify,
485 self.RESULT_KEY: res,
486 self.DATE_KEY: date_str
487 }
488 )
489
490 def handle(self, *args, **kwargs):
491 if len(self.input_dirs) == 0:
492 return
493 result_queue = Queue()
494 process_list = []
495 one_input_dir = None
496 for classify_idx, input_dir in self.input_dirs.items():
497 if one_input_dir is None:
498 one_input_dir = input_dir
499 classify = int(classify_idx.split('_')[0])
500 process = Process(target=self.folder_process, args=(input_dir, classify, result_queue))
501 process_list.append(process)
502
503 wb_dir = os.path.dirname(os.path.dirname(one_input_dir))
504 wb_process = Process(target=self.wb_process, args=(wb_dir, result_queue, ))
505 process_list.append(wb_process)
506
507 for p in process_list:
508 p.start()
509 for p in process_list:
510 p.join()
511
512 self.folder_log.info('{0} [stop safely]'.format(self.log_base))
src/apps/doc/management/commands/folder_ocr_process.py
...@@ -61,13 +61,11 @@ class Command(BaseCommand, LoggerMixin): ...@@ -61,13 +61,11 @@ class Command(BaseCommand, LoggerMixin):
61 def signal_handler(self, sig, frame): 61 def signal_handler(self, sig, frame):
62 self.switch = False # 停止处理文件 62 self.switch = False # 停止处理文件
63 63
64 def license1_process(self, ocr_data, license_summary, classify, res_list, pno, ino, part_idx, img_path): 64 def license1_process(self, ocr_data, license_summary, classify, img_path):
65 # 类别：'0'身份证， '1'居住证 65 # 类别：'0'身份证， '1'居住证
66 license_data = ocr_data.get('data', []) 66 license_data = ocr_data.get('data', [])
67 if not license_data: 67 if not license_data:
68 res_list.append((pno, ino, part_idx, consts.RES_SUCCESS_EMPTY))
69 return 68 return
70 res_list.append((pno, ino, part_idx, consts.RES_SUCCESS))
71 if classify == consts.MVC_CLASSIFY: # 车辆登记证 3/4页结果整合 69 if classify == consts.MVC_CLASSIFY: # 车辆登记证 3/4页结果整合
72 for mvc_dict in license_data: 70 for mvc_dict in license_data:
73 try: 71 try:
...@@ -154,29 +152,21 @@ class Command(BaseCommand, LoggerMixin): ...@@ -154,29 +152,21 @@ class Command(BaseCommand, LoggerMixin):
154 def res_process(self, all_res, classify, excel_path): 152 def res_process(self, all_res, classify, excel_path):
155 try: 153 try:
156 license_summary = {} 154 license_summary = {}
157 res_list = []
158 155
159 if not all_res: 156 if not all_res:
160 return 157 return
161 else: 158 else:
162 for img_path, ocr_res in all_res.items(): 159 for img_path, ocr_res in all_res.items():
163 img_name, pno, ino = self.parse_img_path(img_path) 160 # img_name, pno, ino = self.parse_img_path(img_path)
164 part_idx = 1 161 # part_idx = 1
165 162
166 if isinstance(ocr_res, dict): 163 if isinstance(ocr_res, dict):
167 if ocr_res.get('code') == 1: 164 if ocr_res.get('code') == 1:
168 data_list = ocr_res.get('data', []) 165 data_list = ocr_res.get('data', [])
169 if isinstance(data_list, list): 166 if isinstance(data_list, list):
170 for part_idx, ocr_data in enumerate(data_list): 167 for ocr_data in data_list:
171 part_idx = part_idx + 1 168 # part_idx = part_idx + 1
172 self.license1_process(ocr_data, license_summary, classify, 169 self.license1_process(ocr_data, license_summary, classify, img_path)
173 res_list, pno, ino, part_idx, img_path)
174 else:
175 res_list.append((pno, ino, part_idx, consts.RES_FAILED_3))
176 else:
177 res_list.append((pno, ino, part_idx, consts.RES_FAILED))
178 else:
179 res_list.append((pno, ino, part_idx, consts.RES_FAILED))
180 170
181 wb = BSWorkbook(set(), set(), set(), set(), set()) 171 wb = BSWorkbook(set(), set(), set(), set(), set())
182 wb.simple_license_rebuild(license_summary, consts.DOC_SCHEME_LIST[0]) 172 wb.simple_license_rebuild(license_summary, consts.DOC_SCHEME_LIST[0])
...@@ -217,6 +207,13 @@ class Command(BaseCommand, LoggerMixin): ...@@ -217,6 +207,13 @@ class Command(BaseCommand, LoggerMixin):
217 else: 207 else:
218 self.folder_log.warn('{0} [ocr failed] [img_path={1}]'.format(self.log_base, img_path)) 208 self.folder_log.warn('{0} [ocr failed] [img_path={1}]'.format(self.log_base, img_path))
219 209
210 def images_process(self, img_path_list, classify, excel_path):
211 all_res = {}
212 for img_path in img_path_list:
213 ocr_res = self.ocr_process(img_path, classify)
214 all_res[img_path] = ocr_res
215 self.res_process(all_res, classify, excel_path)
216
220 def pdf_process(self, name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir): 217 def pdf_process(self, name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir):
221 if os.path.exists(path): 218 if os.path.exists(path):
222 try: 219 try:
...@@ -230,11 +227,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -230,11 +227,7 @@ class Command(BaseCommand, LoggerMixin):
230 self.log_base, path, traceback.format_exc())) 227 self.log_base, path, traceback.format_exc()))
231 raise e 228 raise e
232 else: 229 else:
233 all_res = {} 230 self.images_process(pdf_handler.img_path_list, classify, excel_path)
234 for img_path in pdf_handler.img_path_list:
235 ocr_res = self.ocr_process(img_path, classify)
236 all_res[img_path] = ocr_res
237 self.res_process(all_res, classify, excel_path)
238 shutil.move(path, pdf_save_path) 231 shutil.move(path, pdf_save_path)
239 232
240 def tif_process(self, name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir): 233 def tif_process(self, name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir):
...@@ -250,23 +243,18 @@ class Command(BaseCommand, LoggerMixin): ...@@ -250,23 +243,18 @@ class Command(BaseCommand, LoggerMixin):
250 self.log_base, path, traceback.format_exc())) 243 self.log_base, path, traceback.format_exc()))
251 raise e 244 raise e
252 else: 245 else:
253 all_res = {} 246 self.images_process(tiff_handler.img_path_list, classify, excel_path)
254 for img_path in tiff_handler.img_path_list:
255 ocr_res = self.ocr_process(img_path, classify)
256 all_res[img_path] = ocr_res
257 self.res_process(all_res, classify, excel_path)
258 shutil.move(path, tiff_save_path) 247 shutil.move(path, tiff_save_path)
259 248
260 def img_process(self, name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir): 249 def img_process(self, name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir):
261 ocr_res = self.ocr_process(path, classify)
262 all_res = {path: ocr_res}
263
264 try: 250 try:
265 img_save_path, excel_path, _ = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir) 251 img_save_path, excel_path, _ = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir)
266 except Exception as e: 252 except Exception as e:
267 self.folder_log.error('{0} [get path error] [path={1}] [error={2}]'.format( 253 self.folder_log.error('{0} [get path error] [path={1}] [error={2}]'.format(
268 self.log_base, path, traceback.format_exc())) 254 self.log_base, path, traceback.format_exc()))
269 else: 255 else:
256 ocr_res = self.ocr_process(path, classify)
257 all_res = {path: ocr_res}
270 self.res_process(all_res, classify, excel_path) 258 self.res_process(all_res, classify, excel_path)
271 shutil.move(path, img_save_path) 259 shutil.move(path, img_save_path)
272 260
...@@ -312,9 +300,9 @@ class Command(BaseCommand, LoggerMixin): ...@@ -312,9 +300,9 @@ class Command(BaseCommand, LoggerMixin):
312 try: 300 try:
313 if os.path.isfile(path): 301 if os.path.isfile(path):
314 self.folder_log.info('{0} [file start] [path={1}]'.format(self.log_base, path)) 302 self.folder_log.info('{0} [file start] [path={1}]'.format(self.log_base, path))
315 if name.endswith('.pdf'): 303 if name.endswith('.pdf') or name.endswith('.PDF'):
316 self.pdf_process(name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir) 304 self.pdf_process(name, path, classify, img_output_dir, wb_output_dir, pdf_output_dir)
317 elif name.endswith('.tif'): 305 elif name.endswith('.tif') or name.endswith('.TIF'):
318 self.tif_process(name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir) 306 self.tif_process(name, path, classify, img_output_dir, wb_output_dir, tiff_output_dir)
319 else: 307 else:
320 self.img_process(name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir) 308 self.img_process(name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir)
......
src/apps/doc/ocr/wb.py
...@@ -702,6 +702,31 @@ class BSWorkbook(Workbook): ...@@ -702,6 +702,31 @@ class BSWorkbook(Workbook):
702 if field_str is not None: 702 if field_str is not None:
703 count_list.append((field_str, count)) 703 count_list.append((field_str, count))
704 704
705 def ltgt_build(self, label, result_dict):
706 ws = self.create_sheet(label)
707 rebuild_res = {}
708 for key, value in result_dict.items():
709 if isinstance(value, list):
710 value_list = [dict_item.get('words') for dict_item in value]
711 ws.append((key, '、'.join(value_list)))
712 rebuild_res[key] = '、'.join(value_list)
713 elif isinstance(value, dict):
714 if 'words' in value:
715 ws.append((key, value['words']))
716 rebuild_res[key] = value['words']
717 else:
718 for sub_key, sub_value in value.items():
719 if isinstance(sub_value, dict):
720 ws.append(('{0}: {1}'.format(key, sub_key), sub_value.get('words', '')))
721 rebuild_res['{0}: {1}'.format(key, sub_key)] = sub_value.get('words', '')
722 else:
723 ws.append(('{0}: {1}'.format(key, sub_key), sub_value))
724 rebuild_res['{0}: {1}'.format(key, sub_key)] = sub_value
725 else:
726 ws.append((key, value))
727 rebuild_res[key] = value
728 return rebuild_res
729
705 def simple_license_rebuild(self, license_summary, document_scheme): 730 def simple_license_rebuild(self, license_summary, document_scheme):
706 # for ic_license_dict in license_summary.get(consts.IC_CLASSIFY, []): 731 # for ic_license_dict in license_summary.get(consts.IC_CLASSIFY, []):
707 # if ic_license_dict.get('类别') == '1': 732 # if ic_license_dict.get('类别') == '1':
......
src/common/tools/pdf_to_img.py
...@@ -225,3 +225,13 @@ class PDFHandler: ...@@ -225,3 +225,13 @@ class PDFHandler:
225 else: 225 else:
226 self.merge_il(pdf, pno, il) 226 self.merge_il(pdf, pno, il)
227 self.img_count = len(self.img_path_list) 227 self.img_count = len(self.img_path_list)
228
229 def extract_page_image(self):
230 self.img_path_list = []
231 self.xref_set = set()
232 os.makedirs(self.img_dir_path, exist_ok=True)
233 with fitz.Document(self.path) as pdf:
234 for pno in range(pdf.pageCount):
235 page = pdf.loadPage(pno)
236 self.page_to_png(page)
237 self.img_count = len(self.img_path_list)
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!