Skip to content

周伟奇 / bmw-ocr

Go to a project
Toggle navigation
Toggle navigation pinning
0a72fac6 by 周伟奇
Options

fix log

1 parent f5fde5ee
Inline Side-by-side
Showing 3 changed files with 58 additions and 52 deletions
src/apps/doc/management/commands/ocr_process.py
...@@ -68,10 +68,10 @@ class Command(BaseCommand, LoggerMixin): ...@@ -68,10 +68,10 @@ class Command(BaseCommand, LoggerMixin):
68 def get_doc_info(self): 68 def get_doc_info(self):
69 task_str, is_priority = rh.dequeue() 69 task_str, is_priority = rh.dequeue()
70 if task_str is None: 70 if task_str is None:
71 self.cronjob_log.info('{0} [get_doc_info] [queue empty]'.format(self.log_base)) 71 self.online_log.info('{0} [get_doc_info] [queue empty]'.format(self.log_base))
72 return None, None, None 72 return None, None, None
73 73
74 self.cronjob_log.info('{0} [get_doc_info] [task={1}] [is_priority={2}]'.format( 74 self.online_log.info('{0} [get_doc_info] [task={1}] [is_priority={2}]'.format(
75 self.log_base, task_str, is_priority)) 75 self.log_base, task_str, is_priority))
76 try: 76 try:
77 # doc, business_type = self.get_doc_object(task_str) 77 # doc, business_type = self.get_doc_object(task_str)
...@@ -81,11 +81,11 @@ class Command(BaseCommand, LoggerMixin): ...@@ -81,11 +81,11 @@ class Command(BaseCommand, LoggerMixin):
81 doc = doc_class.objects.filter(id=doc_id).first() 81 doc = doc_class.objects.filter(id=doc_id).first()
82 82
83 if doc is None: 83 if doc is None:
84 self.cronjob_log.warn('{0} [get_doc_info] [doc not exist] [task_str={1}] [is_priority={2}]'.format( 84 self.online_log.warn('{0} [get_doc_info] [doc not exist] [task_str={1}] [is_priority={2}]'.format(
85 self.log_base, task_str, is_priority)) 85 self.log_base, task_str, is_priority))
86 return None, None, None 86 return None, None, None
87 elif doc.status != DocStatus.INIT.value: 87 elif doc.status != DocStatus.INIT.value:
88 self.cronjob_log.warn('{0} [get_doc_info] [doc status error] [task_str={1}] [is_priority={2}] ' 88 self.online_log.warn('{0} [get_doc_info] [doc status error] [task_str={1}] [is_priority={2}] '
89 '[doc_status={3}]'.format(self.log_base, task_str, is_priority, doc.status)) 89 '[doc_status={3}]'.format(self.log_base, task_str, is_priority, doc.status))
90 return None, None, None 90 return None, None, None
91 doc.status = DocStatus.PROCESSING.value 91 doc.status = DocStatus.PROCESSING.value
...@@ -93,11 +93,11 @@ class Command(BaseCommand, LoggerMixin): ...@@ -93,11 +93,11 @@ class Command(BaseCommand, LoggerMixin):
93 doc.save() 93 doc.save()
94 except Exception as e: 94 except Exception as e:
95 rh.enqueue([task_str], is_priority) 95 rh.enqueue([task_str], is_priority)
96 self.cronjob_log.error('{0} [process error (get doc info in)] [error={1}]'.format( 96 self.online_log.error('{0} [process error (get doc info in)] [error={1}]'.format(
97 self.log_base, traceback.format_exc())) 97 self.log_base, traceback.format_exc()))
98 raise e 98 raise e
99 else: 99 else:
100 self.cronjob_log.info('{0} [get_doc_info] [db save end] [task_str={1}] [is_priority={2}]'.format( 100 self.online_log.info('{0} [get_doc_info] [db save end] [task_str={1}] [is_priority={2}]'.format(
101 self.log_base, task_str, is_priority)) 101 self.log_base, task_str, is_priority))
102 return doc, business_type, task_str 102 return doc, business_type, task_str
103 103
...@@ -107,14 +107,14 @@ class Command(BaseCommand, LoggerMixin): ...@@ -107,14 +107,14 @@ class Command(BaseCommand, LoggerMixin):
107 # try: 107 # try:
108 # self.edms.download(pdf_path, doc.metadata_version_id) 108 # self.edms.download(pdf_path, doc.metadata_version_id)
109 # except Exception as e: 109 # except Exception as e:
110 # self.cronjob_log.warn('{0} [edms download failed] [times={1}] [pdf_path={2}] ' 110 # self.online_log.warn('{0} [edms download failed] [times={1}] [pdf_path={2}] '
111 # '[error={3}]'.format(self.log_base, times, pdf_path, traceback.format_exc())) 111 # '[error={3}]'.format(self.log_base, times, pdf_path, traceback.format_exc()))
112 # edms_exc = str(e) 112 # edms_exc = str(e)
113 # else: 113 # else:
114 # break 114 # break
115 # else: 115 # else:
116 # raise EDMSException(edms_exc) 116 # raise EDMSException(edms_exc)
117 # self.cronjob_log.info('{0} [edms download success] [pdf_path={1}]'.format(self.log_base, pdf_path)) 117 # self.online_log.info('{0} [edms download success] [pdf_path={1}]'.format(self.log_base, pdf_path))
118 118
119 def bs_process(self, wb, ocr_data, bs_summary, unknown_summary, classify, res_list, pno, ino, part_idx): 119 def bs_process(self, wb, ocr_data, bs_summary, unknown_summary, classify, res_list, pno, ino, part_idx):
120 sheets = ocr_data.get('data', []) 120 sheets = ocr_data.get('data', [])
...@@ -256,7 +256,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -256,7 +256,7 @@ class Command(BaseCommand, LoggerMixin):
256 if ocr_4_response.status_code != 200: 256 if ocr_4_response.status_code != 200:
257 raise OCR4Exception('ocr_4 status code: {0}'.format(ocr_4_response.status_code)) 257 raise OCR4Exception('ocr_4 status code: {0}'.format(ocr_4_response.status_code))
258 except Exception as e: 258 except Exception as e:
259 self.cronjob_log.warn( 259 self.online_log.warn(
260 '{0} [ocr_4 failed] [times={1}] [img_path={2}] [error={3}]'.format( 260 '{0} [ocr_4 failed] [times={1}] [img_path={2}] [error={3}]'.format(
261 self.log_base, times, img_path, traceback.format_exc())) 261 self.log_base, times, img_path, traceback.format_exc()))
262 else: 262 else:
...@@ -268,12 +268,12 @@ class Command(BaseCommand, LoggerMixin): ...@@ -268,12 +268,12 @@ class Command(BaseCommand, LoggerMixin):
268 card_type = ocr_4_res.get('result', {}).get( 268 card_type = ocr_4_res.get('result', {}).get(
269 'idcard_distinguish_result', {}).get('result', -1) 269 'idcard_distinguish_result', {}).get('result', -1)
270 270
271 self.cronjob_log.info( 271 self.online_log.info(
272 '{0} [ocr_4 success] [img_path={1}] [speed_time={2}]'.format( 272 '{0} [ocr_4 success] [img_path={1}] [speed_time={2}]'.format(
273 self.log_base, img_path, speed_time)) 273 self.log_base, img_path, speed_time))
274 break 274 break
275 else: 275 else:
276 self.cronjob_log.warn( 276 self.online_log.warn(
277 '{0} [ocr_4 failed] [img_path={1}]'.format(self.log_base, img_path)) 277 '{0} [ocr_4 failed] [img_path={1}]'.format(self.log_base, img_path))
278 278
279 id_card_dict[consts.IC_TURE_OR_FALSE] = consts.IC_RES_MAPPING.get(card_type) 279 id_card_dict[consts.IC_TURE_OR_FALSE] = consts.IC_RES_MAPPING.get(card_type)
...@@ -474,7 +474,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -474,7 +474,7 @@ class Command(BaseCommand, LoggerMixin):
474 time.sleep(self.sleep_time_doc_get) 474 time.sleep(self.sleep_time_doc_get)
475 continue 475 continue
476 except Exception as e: 476 except Exception as e:
477 self.cronjob_log.error('{0} [process error (get doc info out)] [error={1}]'.format( 477 self.online_log.error('{0} [process error (get doc info out)] [error={1}]'.format(
478 self.log_base, traceback.format_exc())) 478 self.log_base, traceback.format_exc()))
479 error_list.append(1) 479 error_list.append(1)
480 return 480 return
...@@ -492,20 +492,20 @@ class Command(BaseCommand, LoggerMixin): ...@@ -492,20 +492,20 @@ class Command(BaseCommand, LoggerMixin):
492 try: 492 try:
493 if not doc.application_id.startswith(consts.FIXED_APPLICATION_ID_PREFIX): 493 if not doc.application_id.startswith(consts.FIXED_APPLICATION_ID_PREFIX):
494 self.edms.download(pdf_path, doc.metadata_version_id) 494 self.edms.download(pdf_path, doc.metadata_version_id)
495 self.cronjob_log.info('{0} [edms download success] [task={1}] [times={2}] ' 495 self.online_log.info('{0} [edms download success] [task={1}] [times={2}] '
496 '[pdf_path={3}]'.format(self.log_base, task_str, times, pdf_path)) 496 '[pdf_path={3}]'.format(self.log_base, task_str, times, pdf_path))
497 497
498 # 3.PDF文件提取图片 498 # 3.PDF文件提取图片
499 self.cronjob_log.info('{0} [pdf to img start] [task={1}] [times={2}]'.format( 499 self.online_log.info('{0} [pdf to img start] [task={1}] [times={2}]'.format(
500 self.log_base, task_str, times)) 500 self.log_base, task_str, times))
501 start_time = time.time() 501 start_time = time.time()
502 pdf_handler.extract_image() 502 pdf_handler.extract_image()
503 end_time = time.time() 503 end_time = time.time()
504 speed_time = int(end_time - start_time) 504 speed_time = int(end_time - start_time)
505 self.cronjob_log.info('{0} [pdf to img end] [task={1}] [times={2}] [spend_time={3}]'.format( 505 self.online_log.info('{0} [pdf to img end] [task={1}] [times={2}] [spend_time={3}]'.format(
506 self.log_base, task_str, times, speed_time)) 506 self.log_base, task_str, times, speed_time))
507 except Exception as e: 507 except Exception as e:
508 self.cronjob_log.warn('{0} [download or pdf to img failed] [task={1}] [times={2}] ' 508 self.online_log.warn('{0} [download or pdf to img failed] [task={1}] [times={2}] '
509 '[error={3}]'.format(self.log_base, task_str, times, 509 '[error={3}]'.format(self.log_base, task_str, times,
510 traceback.format_exc())) 510 traceback.format_exc()))
511 else: 511 else:
...@@ -515,7 +515,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -515,7 +515,7 @@ class Command(BaseCommand, LoggerMixin):
515 515
516 img_count = len(pdf_handler.img_path_list) 516 img_count = len(pdf_handler.img_path_list)
517 if img_count == 0: 517 if img_count == 0:
518 self.cronjob_log.warn('{0} [pdf to img failed (pdf img empty)] [task={1}]'.format( 518 self.online_log.warn('{0} [pdf to img failed (pdf img empty)] [task={1}]'.format(
519 self.log_base, task_str)) 519 self.log_base, task_str))
520 raise Exception('pdf img empty') 520 raise Exception('pdf img empty')
521 else: 521 else:
...@@ -523,17 +523,17 @@ class Command(BaseCommand, LoggerMixin): ...@@ -523,17 +523,17 @@ class Command(BaseCommand, LoggerMixin):
523 todo_count_dict[task_str] = img_count 523 todo_count_dict[task_str] = img_count
524 for img_path in pdf_handler.img_path_list: 524 for img_path in pdf_handler.img_path_list:
525 while img_queue.full(): 525 while img_queue.full():
526 self.cronjob_log.info('{0} [pdf_2_img_2_queue] [img queue full]'.format(self.log_base)) 526 self.online_log.info('{0} [pdf_2_img_2_queue] [img queue full]'.format(self.log_base))
527 time.sleep(self.sleep_time_img_put) 527 time.sleep(self.sleep_time_img_put)
528 img_queue.put(img_path) 528 img_queue.put(img_path)
529 # except EDMSException as e: 529 # except EDMSException as e:
530 # try: 530 # try:
531 # doc.status = DocStatus.PROCESS_FAILED.value 531 # doc.status = DocStatus.PROCESS_FAILED.value
532 # doc.save() 532 # doc.save()
533 # self.cronjob_log.warn('{0} [process failed (edms download)] [task={1}] [error={2}]'.format( 533 # self.online_log.warn('{0} [process failed (edms download)] [task={1}] [error={2}]'.format(
534 # self.log_base, task_str, traceback.format_exc())) 534 # self.log_base, task_str, traceback.format_exc()))
535 # except Exception as e: 535 # except Exception as e:
536 # self.cronjob_log.error('{0} [process error (db save 1)] [error={1}]'.format( 536 # self.online_log.error('{0} [process error (db save 1)] [error={1}]'.format(
537 # self.log_base, traceback.format_exc())) 537 # self.log_base, traceback.format_exc()))
538 # error_list.append(1) 538 # error_list.append(1)
539 # return 539 # return
...@@ -541,10 +541,10 @@ class Command(BaseCommand, LoggerMixin): ...@@ -541,10 +541,10 @@ class Command(BaseCommand, LoggerMixin):
541 try: 541 try:
542 doc.status = DocStatus.PROCESS_FAILED.value 542 doc.status = DocStatus.PROCESS_FAILED.value
543 doc.save() 543 doc.save()
544 self.cronjob_log.warn('{0} [process failed (pdf_2_img_2_queue)] [task={1}] ' 544 self.online_log.warn('{0} [process failed (pdf_2_img_2_queue)] [task={1}] '
545 '[error={2}]'.format(self.log_base, task_str, traceback.format_exc())) 545 '[error={2}]'.format(self.log_base, task_str, traceback.format_exc()))
546 except Exception as e: 546 except Exception as e:
547 self.cronjob_log.error('{0} [process error (db save)] [error={1}]'.format( 547 self.online_log.error('{0} [process error (db save)] [error={1}]'.format(
548 self.log_base, traceback.format_exc())) 548 self.log_base, traceback.format_exc()))
549 error_list.append(1) 549 error_list.append(1)
550 return 550 return
...@@ -554,12 +554,12 @@ class Command(BaseCommand, LoggerMixin): ...@@ -554,12 +554,12 @@ class Command(BaseCommand, LoggerMixin):
554 try: 554 try:
555 img_path = img_queue.get(block=False) 555 img_path = img_queue.get(block=False)
556 except Exception as e: 556 except Exception as e:
557 # self.cronjob_log.info('{0} [img_2_ocr_1] [queue empty]'.format(self.log_base)) 557 # self.online_log.info('{0} [img_2_ocr_1] [queue empty]'.format(self.log_base))
558 time.sleep(self.sleep_time_img_get) 558 time.sleep(self.sleep_time_img_get)
559 continue 559 continue
560 else: 560 else:
561 try: 561 try:
562 self.cronjob_log.info('{0} [img_2_ocr_1] [get img] [img_path={1}]'.format(self.log_base, img_path)) 562 self.online_log.info('{0} [img_2_ocr_1] [get img] [img_path={1}]'.format(self.log_base, img_path))
563 563
564 for times in range(consts.RETRY_TIMES): 564 for times in range(consts.RETRY_TIMES):
565 try: 565 try:
...@@ -576,23 +576,23 @@ class Command(BaseCommand, LoggerMixin): ...@@ -576,23 +576,23 @@ class Command(BaseCommand, LoggerMixin):
576 if ocr_1_response.status_code != 200: 576 if ocr_1_response.status_code != 200:
577 raise OCR1Exception('ocr_1 status code: {0}'.format(ocr_1_response.status_code)) 577 raise OCR1Exception('ocr_1 status code: {0}'.format(ocr_1_response.status_code))
578 except Exception as e: 578 except Exception as e:
579 self.cronjob_log.warn('{0} [ocr_1 failed] [times={1}] [url={2}] [img_path={3}] ' 579 self.online_log.warn('{0} [ocr_1 failed] [times={1}] [url={2}] [img_path={3}] '
580 '[error={4}]'.format(self.log_base, times, url, img_path, 580 '[error={4}]'.format(self.log_base, times, url, img_path,
581 traceback.format_exc())) 581 traceback.format_exc()))
582 else: 582 else:
583 ocr_1_res = ocr_1_response.json() 583 ocr_1_res = ocr_1_response.json()
584 end_time = time.time() 584 end_time = time.time()
585 speed_time = int(end_time - start_time) 585 speed_time = int(end_time - start_time)
586 self.cronjob_log.info('{0} [ocr_1 success] [img={1}] [url={2}] [speed_time={3}]'.format( 586 self.online_log.info('{0} [ocr_1 success] [img={1}] [url={2}] [speed_time={3}]'.format(
587 self.log_base, img_path, url, speed_time)) 587 self.log_base, img_path, url, speed_time))
588 break 588 break
589 else: 589 else:
590 ocr_1_res = {} 590 ocr_1_res = {}
591 self.cronjob_log.warn('{0} [ocr_1 failed] [img_path={1}] [url={2}]'.format( 591 self.online_log.warn('{0} [ocr_1 failed] [img_path={1}] [url={2}]'.format(
592 self.log_base, img_path, url)) 592 self.log_base, img_path, url))
593 # continue 593 # continue
594 except Exception as e: 594 except Exception as e:
595 self.cronjob_log.error('{0} [process error (ocr fetch)] [img_path={1}] [error={2}]'.format( 595 self.online_log.error('{0} [process error (ocr fetch)] [img_path={1}] [error={2}]'.format(
596 self.log_base, img_path, traceback.format_exc())) 596 self.log_base, img_path, traceback.format_exc()))
597 else: 597 else:
598 try: 598 try:
...@@ -613,7 +613,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -613,7 +613,7 @@ class Command(BaseCommand, LoggerMixin):
613 else: 613 else:
614 todo_count_dict[task_str] = todo_count - 1 614 todo_count_dict[task_str] = todo_count - 1
615 except Exception as e: 615 except Exception as e:
616 self.cronjob_log.error('{0} [process error (store ocr res)] [img_path={1}] [error={2}]'.format( 616 self.online_log.error('{0} [process error (store ocr res)] [img_path={1}] [error={2}]'.format(
617 self.log_base, img_path, traceback.format_exc())) 617 self.log_base, img_path, traceback.format_exc()))
618 618
619 def res_2_wb(self, res_dict, img_queue, finish_queue, lock, error_list): 619 def res_2_wb(self, res_dict, img_queue, finish_queue, lock, error_list):
...@@ -621,11 +621,11 @@ class Command(BaseCommand, LoggerMixin): ...@@ -621,11 +621,11 @@ class Command(BaseCommand, LoggerMixin):
621 try: 621 try:
622 task_str = finish_queue.get(block=False) 622 task_str = finish_queue.get(block=False)
623 except Exception as e: 623 except Exception as e:
624 # self.cronjob_log.info('{0} [res_2_wb] [queue empty]'.format(self.log_base)) 624 # self.online_log.info('{0} [res_2_wb] [queue empty]'.format(self.log_base))
625 time.sleep(self.sleep_time_task_get) 625 time.sleep(self.sleep_time_task_get)
626 continue 626 continue
627 else: 627 else:
628 self.cronjob_log.info('{0} [res_2_wb] [get task] [task={1}]'.format(self.log_base, task_str)) 628 self.online_log.info('{0} [res_2_wb] [get task] [task={1}]'.format(self.log_base, task_str))
629 ocr_1_res = res_dict.pop(task_str, {}) 629 ocr_1_res = res_dict.pop(task_str, {})
630 630
631 business_type, doc_id_str = task_str.split(consts.SPLIT_STR) 631 business_type, doc_id_str = task_str.split(consts.SPLIT_STR)
...@@ -638,7 +638,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -638,7 +638,7 @@ class Command(BaseCommand, LoggerMixin):
638 try: 638 try:
639 doc = doc_class.objects.filter(id=doc_id).first() 639 doc = doc_class.objects.filter(id=doc_id).first()
640 except Exception as e: 640 except Exception as e:
641 self.cronjob_log.error('{0} [process error (db filter)] [task={1}] [error={2}]'.format( 641 self.online_log.error('{0} [process error (db filter)] [task={1}] [error={2}]'.format(
642 self.log_base, task_str, traceback.format_exc())) 642 self.log_base, task_str, traceback.format_exc()))
643 else: 643 else:
644 try: 644 try:
...@@ -663,14 +663,14 @@ class Command(BaseCommand, LoggerMixin): ...@@ -663,14 +663,14 @@ class Command(BaseCommand, LoggerMixin):
663 ocr_data_list = res.get('data', []) 663 ocr_data_list = res.get('data', [])
664 if not isinstance(ocr_data_list, list): 664 if not isinstance(ocr_data_list, list):
665 res_list.append((pno, ino, part_idx, consts.RES_FAILED_3)) 665 res_list.append((pno, ino, part_idx, consts.RES_FAILED_3))
666 self.cronjob_log.warn('{0} [ocr_1 res error] [img={1}]'.format(self.log_base, img_path)) 666 self.online_log.warn('{0} [ocr_1 res error] [img={1}]'.format(self.log_base, img_path))
667 else: 667 else:
668 for part_idx, ocr_data in enumerate(ocr_data_list): 668 for part_idx, ocr_data in enumerate(ocr_data_list):
669 part_idx = part_idx + 1 669 part_idx = part_idx + 1
670 classify = ocr_data.get('classify') 670 classify = ocr_data.get('classify')
671 if classify is None: 671 if classify is None:
672 res_list.append((pno, ino, part_idx, consts.RES_FAILED_3)) 672 res_list.append((pno, ino, part_idx, consts.RES_FAILED_3))
673 self.cronjob_log.warn('{0} [ocr_1 res error] [img={1}]'.format( 673 self.online_log.warn('{0} [ocr_1 res error] [img={1}]'.format(
674 self.log_base, img_path)) 674 self.log_base, img_path))
675 continue 675 continue
676 elif classify in consts.OTHER_CLASSIFY_SET: # 其他类 676 elif classify in consts.OTHER_CLASSIFY_SET: # 其他类
...@@ -699,14 +699,14 @@ class Command(BaseCommand, LoggerMixin): ...@@ -699,14 +699,14 @@ class Command(BaseCommand, LoggerMixin):
699 if ocr_2_response.status_code != 200: 699 if ocr_2_response.status_code != 200:
700 raise OCR2Exception('ocr_2 status code: {0}'.format(ocr_2_response.status_code)) 700 raise OCR2Exception('ocr_2 status code: {0}'.format(ocr_2_response.status_code))
701 except Exception as e: 701 except Exception as e:
702 self.cronjob_log.warn( 702 self.online_log.warn(
703 '{0} [ocr_2 failed] [times={1}] [img_path={2}] [error={3}]'.format( 703 '{0} [ocr_2 failed] [times={1}] [img_path={2}] [error={3}]'.format(
704 self.log_base, times, img_path, traceback.format_exc())) 704 self.log_base, times, img_path, traceback.format_exc()))
705 else: 705 else:
706 ocr_2_res = json.loads(ocr_2_response.text) 706 ocr_2_res = json.loads(ocr_2_response.text)
707 end_time = time.time() 707 end_time = time.time()
708 speed_time = int(end_time - start_time) 708 speed_time = int(end_time - start_time)
709 self.cronjob_log.info( 709 self.online_log.info(
710 '{0} [ocr_2 success] [img={1}] [speed_time={2}]'.format( 710 '{0} [ocr_2 success] [img={1}] [speed_time={2}]'.format(
711 self.log_base, img_path, speed_time)) 711 self.log_base, img_path, speed_time))
712 712
...@@ -727,15 +727,15 @@ class Command(BaseCommand, LoggerMixin): ...@@ -727,15 +727,15 @@ class Command(BaseCommand, LoggerMixin):
727 break 727 break
728 else: 728 else:
729 res_list.append((pno, ino, part_idx, consts.RES_FAILED_2)) 729 res_list.append((pno, ino, part_idx, consts.RES_FAILED_2))
730 self.cronjob_log.warn( 730 self.online_log.warn(
731 '{0} [ocr_2 failed] [img_path={1}]'.format(self.log_base, img_path)) 731 '{0} [ocr_2 failed] [img_path={1}]'.format(self.log_base, img_path))
732 else: # 流水处理 732 else: # 流水处理
733 self.bs_process(wb, ocr_data, bs_summary, unknown_summary, classify, res_list, pno, ino, part_idx) 733 self.bs_process(wb, ocr_data, bs_summary, unknown_summary, classify, res_list, pno, ino, part_idx)
734 else: 734 else:
735 res_list.append((pno, ino, part_idx, consts.RES_FAILED_1)) 735 res_list.append((pno, ino, part_idx, consts.RES_FAILED_1))
736 self.cronjob_log.info('{0} [ocr_1 res error] [img={1}]'.format(self.log_base, img_path)) 736 self.online_log.info('{0} [ocr_1 res error] [img={1}]'.format(self.log_base, img_path))
737 737
738 self.cronjob_log.info('{0} [task={1}] [bs_summary={2}] [unknown_summary={3}] ' 738 self.online_log.info('{0} [task={1}] [bs_summary={2}] [unknown_summary={3}] '
739 '[license_summary={4}]'.format(self.log_base, task_str, bs_summary, 739 '[license_summary={4}]'.format(self.log_base, task_str, bs_summary,
740 unknown_summary, license_summary)) 740 unknown_summary, license_summary))
741 741
...@@ -748,21 +748,21 @@ class Command(BaseCommand, LoggerMixin): ...@@ -748,21 +748,21 @@ class Command(BaseCommand, LoggerMixin):
748 748
749 self.bs_log.info('[task={0}] [bs_summary={1}]'.format(task_str, merged_bs_summary)) 749 self.bs_log.info('[task={0}] [bs_summary={1}]'.format(task_str, merged_bs_summary))
750 750
751 self.cronjob_log.info('{0} [task={1}] [merged_bs_summary={2}] [unknown_summary={3}] ' 751 self.online_log.info('{0} [task={1}] [merged_bs_summary={2}] [unknown_summary={3}] '
752 '[res_list={4}]'.format(self.log_base, task_str, merged_bs_summary, 752 '[res_list={4}]'.format(self.log_base, task_str, merged_bs_summary,
753 unknown_summary, res_list)) 753 unknown_summary, res_list))
754 del unknown_summary 754 del unknown_summary
755 755
756 except Exception as e: 756 except Exception as e:
757 757
758 self.cronjob_log.warn('{0} [process failed (res conformity)] [task={1}] [error={2}]'.format( 758 self.online_log.warn('{0} [process failed (res conformity)] [task={1}] [error={2}]'.format(
759 self.log_base, task_str, traceback.format_exc())) 759 self.log_base, task_str, traceback.format_exc()))
760 760
761 try: 761 try:
762 doc.status = DocStatus.PROCESS_FAILED.value 762 doc.status = DocStatus.PROCESS_FAILED.value
763 doc.save() 763 doc.save()
764 except Exception as e: 764 except Exception as e:
765 self.cronjob_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format( 765 self.online_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format(
766 self.log_base, task_str, traceback.format_exc())) 766 self.log_base, task_str, traceback.format_exc()))
767 767
768 else: 768 else:
...@@ -776,14 +776,14 @@ class Command(BaseCommand, LoggerMixin): ...@@ -776,14 +776,14 @@ class Command(BaseCommand, LoggerMixin):
776 776
777 except Exception as e: 777 except Exception as e:
778 778
779 self.cronjob_log.warn('{0} [process failed (wb rebuild)] [task={1}] [error={2}]'.format( 779 self.online_log.warn('{0} [process failed (wb rebuild)] [task={1}] [error={2}]'.format(
780 self.log_base, task_str, traceback.format_exc())) 780 self.log_base, task_str, traceback.format_exc()))
781 781
782 try: 782 try:
783 doc.status = DocStatus.PROCESS_FAILED.value 783 doc.status = DocStatus.PROCESS_FAILED.value
784 doc.save() 784 doc.save()
785 except Exception as e: 785 except Exception as e:
786 self.cronjob_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format( 786 self.online_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format(
787 self.log_base, task_str, traceback.format_exc())) 787 self.log_base, task_str, traceback.format_exc()))
788 788
789 else: 789 else:
...@@ -793,7 +793,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -793,7 +793,7 @@ class Command(BaseCommand, LoggerMixin):
793 try: 793 try:
794 self.edms.upload(excel_path, doc, business_type) 794 self.edms.upload(excel_path, doc, business_type)
795 except Exception as e: 795 except Exception as e:
796 self.cronjob_log.warn( 796 self.online_log.warn(
797 '{0} [edms upload failed] [times={1}] [task={2}] [error={3}]'.format( 797 '{0} [edms upload failed] [times={1}] [task={2}] [error={3}]'.format(
798 self.log_base, times, task_str, traceback.format_exc())) 798 self.log_base, times, task_str, traceback.format_exc()))
799 edms_exc = str(e) 799 edms_exc = str(e)
...@@ -803,11 +803,11 @@ class Command(BaseCommand, LoggerMixin): ...@@ -803,11 +803,11 @@ class Command(BaseCommand, LoggerMixin):
803 raise EDMSException(edms_exc) 803 raise EDMSException(edms_exc)
804 except Exception as e: 804 except Exception as e:
805 doc.status = DocStatus.UPLOAD_FAILED.value 805 doc.status = DocStatus.UPLOAD_FAILED.value
806 self.cronjob_log.warn('{0} [process failed (edms upload)] [task={1}] [error={2}]'.format( 806 self.online_log.warn('{0} [process failed (edms upload)] [task={1}] [error={2}]'.format(
807 self.log_base, task_str, traceback.format_exc())) 807 self.log_base, task_str, traceback.format_exc()))
808 else: 808 else:
809 doc.status = DocStatus.COMPLETE.value 809 doc.status = DocStatus.COMPLETE.value
810 self.cronjob_log.info('{0} [edms upload success] [task={1}]'.format(self.log_base, task_str)) 810 self.online_log.info('{0} [edms upload success] [task={1}]'.format(self.log_base, task_str))
811 finally: 811 finally:
812 try: 812 try:
813 doc.end_time = timezone.now() 813 doc.end_time = timezone.now()
...@@ -817,10 +817,10 @@ class Command(BaseCommand, LoggerMixin): ...@@ -817,10 +817,10 @@ class Command(BaseCommand, LoggerMixin):
817 setattr(doc, field, count) 817 setattr(doc, field, count)
818 doc.save() 818 doc.save()
819 except Exception as e: 819 except Exception as e:
820 self.cronjob_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format( 820 self.online_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format(
821 self.log_base, task_str, traceback.format_exc())) 821 self.log_base, task_str, traceback.format_exc()))
822 else: 822 else:
823 self.cronjob_log.info('{0} [process complete] [task={1}]'.format(self.log_base, task_str)) 823 self.online_log.info('{0} [process complete] [task={1}]'.format(self.log_base, task_str))
824 os.remove(excel_path) 824 os.remove(excel_path)
825 finally: 825 finally:
826 pass 826 pass
...@@ -837,7 +837,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -837,7 +837,7 @@ class Command(BaseCommand, LoggerMixin):
837 pdf_path = os.path.join(doc_data_path, '{0}.pdf'.format(doc_id_str)) 837 pdf_path = os.path.join(doc_data_path, '{0}.pdf'.format(doc_id_str))
838 os.remove(pdf_path) 838 os.remove(pdf_path)
839 except Exception as e: 839 except Exception as e:
840 self.cronjob_log.error('{0} [process error (pdf & img remove)] [task={1}] [error={2}]'.format( 840 self.online_log.error('{0} [process error (pdf & img remove)] [task={1}] [error={2}]'.format(
841 self.log_base, task_str, traceback.format_exc())) 841 self.log_base, task_str, traceback.format_exc()))
842 842
843 def handle(self, *args, **kwargs): 843 def handle(self, *args, **kwargs):
...@@ -867,4 +867,4 @@ class Command(BaseCommand, LoggerMixin): ...@@ -867,4 +867,4 @@ class Command(BaseCommand, LoggerMixin):
867 for p in process_list: 867 for p in process_list:
868 p.join() 868 p.join()
869 869
870 self.cronjob_log.info('{0} [stop safely]'.format(self.log_base)) 870 self.online_log.info('{0} [stop safely]'.format(self.log_base))
......
src/common/mixins.py
...@@ -38,7 +38,7 @@ class GenericExceptionMixin: ...@@ -38,7 +38,7 @@ class GenericExceptionMixin:
38 class LoggerMixin: 38 class LoggerMixin:
39 running_log = logging.getLogger('running') 39 running_log = logging.getLogger('running')
40 exception_log = logging.getLogger('exception') 40 exception_log = logging.getLogger('exception')
41 cronjob_log = logging.getLogger('cronjob') 41 online_log = logging.getLogger('online')
42 folder_log = logging.getLogger('folder') 42 folder_log = logging.getLogger('folder')
43 bs_log = logging.getLogger('bs') 43 bs_log = logging.getLogger('bs')
44 license_log = logging.getLogger('license') 44 license_log = logging.getLogger('license')
......
src/settings/conf/logging.conf
1 [loggers] 1 [loggers]
2 ; online --> 线上OCR日志：ocr_process.py
3 ; folder --> 线下OCR日志：folder_ocr_process.py
4 ; bs --> 线上流水识别结果日志，用于生成每日申请号-版式报表：bs_statistics.py
5 ; license --> 线上证件识别结果日志，用于偶尔生成结果报表：license_statistics.py
6 ; idcard --> 线上身份证识别结果日志，用于生成每日、每月身份证报表：idcard_daily.py、idcard_monthly.py
7 ; compare --> celery compare日志
2 keys=root, running, exception, online, folder, bs, license, idcard, compare, django.db.backends 8 keys=root, running, exception, online, folder, bs, license, idcard, compare, django.db.backends
3 9
4 [handlers] 10 [handlers]
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!