Skip to content

周伟奇 / bmw-ocr

Go to a project
Toggle navigation
Toggle navigation pinning
fc360367 by 周伟奇
Options

report part 1

1 parent 9b2170ec
Inline Side-by-side
Showing 3 changed files with 186 additions and 3 deletions
src/apps/doc/management/commands/ocr_process.py
...@@ -20,10 +20,10 @@ from common.tools.file_tools import write_zip_file ...@@ -20,10 +20,10 @@ from common.tools.file_tools import write_zip_file
20 from common.tools.pdf_to_img import PDFHandler 20 from common.tools.pdf_to_img import PDFHandler
21 from apps.doc import consts 21 from apps.doc import consts
22 from apps.doc.ocr.edms import EDMS, rh 22 from apps.doc.ocr.edms import EDMS, rh
23 from apps.doc.named_enum import KeywordsType 23 from apps.doc.named_enum import KeywordsType, FailureReason, WorkflowName, ProcessName, RequestTeam, RequestTrigger
24 from apps.doc.exceptions import EDMSException, OCR1Exception, OCR2Exception, OCR4Exception 24 from apps.doc.exceptions import EDMSException, OCR1Exception, OCR2Exception, OCR4Exception
25 from apps.doc.ocr.wb import BSWorkbook 25 from apps.doc.ocr.wb import BSWorkbook
26 from apps.doc.models import DocStatus, HILDoc, AFCDoc, Keywords, HILOCRResult, AFCOCRResult 26 from apps.doc.models import DocStatus, HILDoc, AFCDoc, Keywords, HILOCRResult, AFCOCRResult, HILOCRReport, AFCOCRReport
27 from celery_compare.tasks import compare 27 from celery_compare.tasks import compare
28 28
29 29
...@@ -604,6 +604,24 @@ class Command(BaseCommand, LoggerMixin): ...@@ -604,6 +604,24 @@ class Command(BaseCommand, LoggerMixin):
604 # return 604 # return
605 except Exception as e: 605 except Exception as e:
606 try: 606 try:
607 end_time = timezone.now()
608 report_table = HILOCRReport if business_type == consts.HIL_PREFIX else AFCOCRReport
609 report_table.objects.create(
610 case_number=doc.application_id,
611 request_team=RequestTeam.get_value(doc.document_scheme, 0),
612 request_trigger=RequestTrigger.get_value(doc.data_source, 0),
613 input_file=doc.document_name,
614 transaction_start=doc.start_time,
615 transaction_end=end_time,
616 successful_at_this_level=False,
617 failure_reason=FailureReason.PDF.value,
618 process_name=ProcessName.ALL.value,
619 )
620 except Exception as e:
621 self.online_log.error('{0} [process error (report db save)] [error={1}]'.format(
622 self.log_base, traceback.format_exc()))
623
624 try:
607 doc.status = DocStatus.PROCESS_FAILED.value 625 doc.status = DocStatus.PROCESS_FAILED.value
608 doc.save() 626 doc.save()
609 self.online_log.warn('{0} [process failed (pdf_2_img_2_queue)] [task={1}] ' 627 self.online_log.warn('{0} [process failed (pdf_2_img_2_queue)] [task={1}] '
...@@ -702,12 +720,19 @@ class Command(BaseCommand, LoggerMixin): ...@@ -702,12 +720,19 @@ class Command(BaseCommand, LoggerMixin):
702 720
703 try: 721 try:
704 doc = doc_class.objects.filter(id=doc_id).first() 722 doc = doc_class.objects.filter(id=doc_id).first()
723 # report_dict = {
724 # 'process': None or pdf or excel or edms
725 # 'idcard': True or False,
726 # 'bs': None or normal or mobile,
727 # }
728 report_list = [None, False, None]
705 except Exception as e: 729 except Exception as e:
706 self.online_log.error('{0} [process error (db filter)] [task={1}] [error={2}]'.format( 730 self.online_log.error('{0} [process error (db filter)] [task={1}] [error={2}]'.format(
707 self.log_base, task_str, traceback.format_exc())) 731 self.log_base, task_str, traceback.format_exc()))
708 else: 732 else:
709 try: 733 try:
710 # 4.OCR结果并且构建excel文件 734 # 4.OCR结果并且构建excel文件
735 bs_classify_set = set()
711 bs_summary = {} 736 bs_summary = {}
712 unknown_summary = {} 737 unknown_summary = {}
713 license_summary = {} 738 license_summary = {}
...@@ -795,6 +820,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -795,6 +820,7 @@ class Command(BaseCommand, LoggerMixin):
795 self.online_log.warn( 820 self.online_log.warn(
796 '{0} [ocr_2 failed] [img_path={1}]'.format(self.log_base, img_path)) 821 '{0} [ocr_2 failed] [img_path={1}]'.format(self.log_base, img_path))
797 else: # 流水处理 822 else: # 流水处理
823 bs_classify_set.add(classify)
798 self.bs_process(wb, ocr_data, bs_summary, unknown_summary, classify, res_list, pno, ino, part_idx) 824 self.bs_process(wb, ocr_data, bs_summary, unknown_summary, classify, res_list, pno, ino, part_idx)
799 else: 825 else:
800 res_list.append((pno, ino, part_idx, consts.RES_FAILED_1)) 826 res_list.append((pno, ino, part_idx, consts.RES_FAILED_1))
...@@ -806,8 +832,15 @@ class Command(BaseCommand, LoggerMixin): ...@@ -806,8 +832,15 @@ class Command(BaseCommand, LoggerMixin):
806 # self.license_log.info('[task={0}] [license_summary={1}]'.format(task_str, license_summary)) 832 # self.license_log.info('[task={0}] [license_summary={1}]'.format(task_str, license_summary))
807 idcard_list = license_summary.get(consts.IC_CLASSIFY) 833 idcard_list = license_summary.get(consts.IC_CLASSIFY)
808 if idcard_list: 834 if idcard_list:
835 report_list[1] = True
809 self.idcard_log.info('[task={0}] [idcard={1}]'.format(task_str, idcard_list)) 836 self.idcard_log.info('[task={0}] [idcard={1}]'.format(task_str, idcard_list))
810 837
838 if len(bs_classify_set) > 0:
839 if consts.ALI_WECHART_CLASSIFY & bs_classify_set:
840 report_list[2] = WorkflowName.MOBILE.value
841 else:
842 report_list[2] = WorkflowName.NORMAL.value
843
811 merged_bs_summary = self.rebuild_bs_summary(bs_summary, unknown_summary) 844 merged_bs_summary = self.rebuild_bs_summary(bs_summary, unknown_summary)
812 del unknown_summary 845 del unknown_summary
813 846
...@@ -821,6 +854,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -821,6 +854,7 @@ class Command(BaseCommand, LoggerMixin):
821 854
822 except Exception as e: 855 except Exception as e:
823 856
857 report_list[0] = FailureReason.EXCEL.value
824 self.online_log.warn('{0} [process failed (res conformity)] [task={1}] [error={2}]'.format( 858 self.online_log.warn('{0} [process failed (res conformity)] [task={1}] [error={2}]'.format(
825 self.log_base, task_str, traceback.format_exc())) 859 self.log_base, task_str, traceback.format_exc()))
826 860
...@@ -842,6 +876,7 @@ class Command(BaseCommand, LoggerMixin): ...@@ -842,6 +876,7 @@ class Command(BaseCommand, LoggerMixin):
842 876
843 except Exception as e: 877 except Exception as e:
844 878
879 report_list[0] = FailureReason.EXCEL.value
845 self.online_log.warn('{0} [process failed (wb rebuild)] [task={1}] [error={2}]'.format( 880 self.online_log.warn('{0} [process failed (wb rebuild)] [task={1}] [error={2}]'.format(
846 self.log_base, task_str, traceback.format_exc())) 881 self.log_base, task_str, traceback.format_exc()))
847 882
...@@ -868,12 +903,17 @@ class Command(BaseCommand, LoggerMixin): ...@@ -868,12 +903,17 @@ class Command(BaseCommand, LoggerMixin):
868 else: 903 else:
869 raise EDMSException(edms_exc) 904 raise EDMSException(edms_exc)
870 except Exception as e: 905 except Exception as e:
906
907 report_list[0] = FailureReason.EDMS.value
871 doc.status = DocStatus.UPLOAD_FAILED.value 908 doc.status = DocStatus.UPLOAD_FAILED.value
872 self.online_log.warn('{0} [process failed (edms upload)] [task={1}] [error={2}]'.format( 909 self.online_log.warn('{0} [process failed (edms upload)] [task={1}] [error={2}]'.format(
873 self.log_base, task_str, traceback.format_exc())) 910 self.log_base, task_str, traceback.format_exc()))
911
874 else: 912 else:
913
875 doc.status = DocStatus.COMPLETE.value 914 doc.status = DocStatus.COMPLETE.value
876 self.online_log.info('{0} [edms upload success] [task={1}]'.format(self.log_base, task_str)) 915 self.online_log.info('{0} [edms upload success] [task={1}]'.format(self.log_base, task_str))
916
877 finally: 917 finally:
878 try: 918 try:
879 doc.end_time = timezone.now() 919 doc.end_time = timezone.now()
...@@ -938,6 +978,74 @@ class Command(BaseCommand, LoggerMixin): ...@@ -938,6 +978,74 @@ class Command(BaseCommand, LoggerMixin):
938 else: 978 else:
939 self.online_log.info('{0} [comparison info send success] [task={1}] ' 979 self.online_log.info('{0} [comparison info send success] [task={1}] '
940 '[res_id={2}]'.format(self.log_base, task_str, res_obj.id)) 980 '[res_id={2}]'.format(self.log_base, task_str, res_obj.id))
981 finally:
982 # report_dict = {
983 # 'process': None or pdf or excel or edms
984 # 'idcard': True or False,
985 # 'bs': None or normal or mobile,
986 # }
987
988 end_time = timezone.now()
989 report_table = HILOCRReport if business_type == consts.HIL_PREFIX else AFCOCRReport
990
991 try:
992 if report_list[0] is None:
993 report_table.objects.create(
994 case_number=doc.application_id,
995 request_team=RequestTeam.get_value(doc.document_scheme, 0),
996 request_trigger=RequestTrigger.get_value(doc.data_source, 0),
997 input_file=doc.document_name,
998 transaction_start=doc.start_time,
999 transaction_end=end_time,
1000 process_name=ProcessName.ALL.value,
1001 )
1002 else:
1003 report_table.objects.create(
1004 case_number=doc.application_id,
1005 request_team=RequestTeam.get_value(doc.document_scheme, 0),
1006 request_trigger=RequestTrigger.get_value(doc.data_source, 0),
1007 input_file=doc.document_name,
1008 transaction_start=doc.start_time,
1009 transaction_end=end_time,
1010 successful_at_this_level=False,
1011 failure_reason=report_list[0],
1012 process_name=ProcessName.ALL.value,
1013 )
1014 except Exception as e:
1015 self.online_log.error('{0} [process error (report db save)] [error={1}]'.format(
1016 self.log_base, traceback.format_exc()))
1017
1018 try:
1019 if report_list[1]:
1020 report_table.objects.create(
1021 case_number=doc.application_id,
1022 request_team=RequestTeam.CONTROLLING.value,
1023 request_trigger=RequestTrigger.DOCUPLOAD.value,
1024 input_file=doc.document_name,
1025 transaction_start=doc.start_time,
1026 transaction_end=end_time,
1027 process_name=ProcessName.IDCARD.value,
1028 )
1029 except Exception as e:
1030 self.online_log.error('{0} [process error (report db save)] [error={1}]'.format(
1031 self.log_base, traceback.format_exc()))
1032
1033 try:
1034 if report_list[2] is not None:
1035 report_table.objects.create(
1036 case_number=doc.application_id,
1037 request_team=RequestTeam.get_value(doc.document_scheme, 0),
1038 request_trigger=RequestTrigger.DOCUPLOAD.value,
1039 input_file=doc.document_name,
1040 transaction_start=doc.start_time,
1041 transaction_end=end_time,
1042 process_name=ProcessName.BS.value,
1043 workflow_name=report_list[2],
1044 )
1045 except Exception as e:
1046 self.online_log.error('{0} [process error (report db save)] [error={1}]'.format(
1047 self.log_base, traceback.format_exc()))
1048
941 finally: 1049 finally:
942 try: 1050 try:
943 img_save_path = os.path.join(doc_data_path, 'img') 1051 img_save_path = os.path.join(doc_data_path, 'img')
......
src/apps/doc/models.py
1 from django.db import models 1 from django.db import models
2 from .named_enum import DocStatus, KeywordsType, RetryStep 2 from .named_enum import DocStatus, KeywordsType, RequestTeam, RequestTrigger, FailureReason, ProcessName, WorkflowName
3 3
4 # Create your models here. 4 # Create your models here.
5 5
...@@ -267,3 +267,45 @@ class HILOCRResult(models.Model): ...@@ -267,3 +267,45 @@ class HILOCRResult(models.Model):
267 managed = False 267 managed = False
268 db_table = 'hil_ocr_result' 268 db_table = 'hil_ocr_result'
269 269
270
271 # OCR Report
272 class HILOCRReport(models.Model):
273 id = models.AutoField(primary_key=True, verbose_name="id") # 主键
274 case_number = models.CharField(max_length=64, verbose_name="申请id")
275 request_team = models.SmallIntegerField(default=RequestTeam.ACCEPTANCE.value, verbose_name="来源")
276 request_trigger = models.SmallIntegerField(default=RequestTrigger.POS.value, verbose_name="触发者")
277 input_file = models.CharField(max_length=255, verbose_name="文件名")
278 transaction_start = models.DateTimeField(null=True, verbose_name='开始时间') # 索引
279 transaction_end = models.DateTimeField(null=True, verbose_name='结束时间')
280 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功")
281 failure_reason = models.SmallIntegerField(null=True, verbose_name="失败原因")
282 process_name = models.SmallIntegerField(default=ProcessName.ALL.value, verbose_name="流程名称")
283 total_fields = models.IntegerField(null=True, verbose_name='比对字段数目')
284 workflow_name = models.SmallIntegerField(null=True, verbose_name="工作流程")
285
286 class Meta:
287 managed = False
288 db_table = 'hil_ocr_report'
289
290
291 class AFCOCRReport(models.Model):
292 id = models.AutoField(primary_key=True, verbose_name="id") # 主键
293 case_number = models.CharField(max_length=64, verbose_name="申请id")
294 request_team = models.SmallIntegerField(default=RequestTeam.ACCEPTANCE.value, verbose_name="来源")
295 request_trigger = models.SmallIntegerField(default=RequestTrigger.POS.value, verbose_name="触发者")
296 input_file = models.CharField(max_length=255, verbose_name="文件名")
297 transaction_start = models.DateTimeField(null=True, verbose_name='开始时间') # 索引
298 transaction_end = models.DateTimeField(null=True, verbose_name='结束时间')
299 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功")
300 failure_reason = models.SmallIntegerField(null=True, verbose_name="失败原因")
301 process_name = models.SmallIntegerField(default=ProcessName.ALL.value, verbose_name="流程名称")
302 total_fields = models.IntegerField(null=True, verbose_name='比对字段数目')
303 workflow_name = models.SmallIntegerField(null=True, verbose_name="工作流程")
304
305 class Meta:
306 managed = False
307 db_table = 'afc_ocr_report'
308 situ_db_label = 'afc'
309
310
311
......
src/apps/doc/named_enum.py
...@@ -19,3 +19,36 @@ class KeywordsType(NamedEnum): ...@@ -19,3 +19,36 @@ class KeywordsType(NamedEnum):
19 SALARY = (1, '薪资') 19 SALARY = (1, '薪资')
20 LOAN = (2, '贷款') 20 LOAN = (2, '贷款')
21 ALI_WECHART = (3, '微信/支付宝') 21 ALI_WECHART = (3, '微信/支付宝')
22
23
24 class RequestTeam(NamedEnum):
25 ACCEPTANCE = (0, 'ACCEPTANCE')
26 SETTLEMENT = (1, 'SETTLEMENT')
27 CONTRACTMANAGEMENT = (2, 'CONTRACTMANAGEMENT')
28 CONTROLLING = (3, 'CONTROLLING')
29
30
31 class RequestTrigger(NamedEnum):
32 POS = (0, 'POS')
33 EAPP = (1, 'EAPP')
34 ECONTRACT = (2, 'ECONTRACT')
35 DOCUPLOAD = (3, 'Document Upload')
36
37
38 class FailureReason(NamedEnum):
39 PDF = (0, 'PDF处理失败')
40 EXCEL = (1, '构建excel失败')
41 EDMS = (2, 'EDMS上传失败')
42
43
44 class ProcessName(NamedEnum):
45 ALL = (0, 'S1_All_DocumentUpload')
46 BS = (1, 'S1_CA_BankStatementCalculation')
47 IDCARD = (2, 'F2_IDReport')
48 DDA = (3, 'CL_S1_DDAConsolidation')
49
50
51 class WorkflowName(NamedEnum):
52 NORMAL = (0, 'Normal BS')
53 MOBILE = (0, 'Mobile BS')
54
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!