Skip to content

周伟奇 / bmw-ocr

Go to a project
Toggle navigation
Toggle navigation pinning
e1cd63be by 周伟奇
Options

add report part 2

1 parent 828d28c7
Inline Side-by-side
Showing 6 changed files with 259 additions and 12 deletions
src/apps/doc/consts.py
...@@ -1084,6 +1084,7 @@ ID_TYPE_COMPARE = { ...@@ -1084,6 +1084,7 @@ ID_TYPE_COMPARE = {
1084 } 1084 }
1085 1085
1086 ID_RES = 'idNumResult' 1086 ID_RES = 'idNumResult'
1087 SECOND_ID_FIELD = 'secondIdNum'
1087 SECOND_ID_RES = 'secondIdNumResult' 1088 SECOND_ID_RES = 'secondIdNumResult'
1088 1089
1089 SECOND_ITPRC = [ 1090 SECOND_ITPRC = [
......
src/apps/doc/models.py
...@@ -312,15 +312,15 @@ class HILCompareReport(models.Model): ...@@ -312,15 +312,15 @@ class HILCompareReport(models.Model):
312 id = models.AutoField(primary_key=True, verbose_name="id") # 主键 312 id = models.AutoField(primary_key=True, verbose_name="id") # 主键
313 case_number = models.CharField(max_length=64, verbose_name="申请id") 313 case_number = models.CharField(max_length=64, verbose_name="申请id")
314 request_team = models.SmallIntegerField(default=RequestTeam.ACCEPTANCE.value, verbose_name="来源") 314 request_team = models.SmallIntegerField(default=RequestTeam.ACCEPTANCE.value, verbose_name="来源")
315 request_trigger = models.SmallIntegerField(default=RequestTrigger.POS.value, verbose_name="触发者") 315 request_trigger = models.SmallIntegerField(default=RequestTrigger.SUBMITING.value, verbose_name="触发者")
316 input_file = models.CharField(max_length=255, verbose_name="文件名", null=True) 316 input_file = models.CharField(max_length=255, verbose_name="文件名", null=True)
317 transaction_start = models.DateTimeField(verbose_name='开始时间') # 索引 317 transaction_start = models.DateTimeField(verbose_name='开始时间') # 索引
318 transaction_end = models.DateTimeField(verbose_name='结束时间') 318 transaction_end = models.DateTimeField(verbose_name='结束时间')
319 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功") 319 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功")
320 failure_reason = models.TextField(null=True, verbose_name="失败原因") 320 failure_reason = models.TextField(null=True, verbose_name="失败原因")
321 process_name = models.SmallIntegerField(default=ProcessName.ALL.value, verbose_name="流程名称") 321 process_name = models.SmallIntegerField(default=ProcessName.CACOMPARE.value, verbose_name="流程名称")
322 total_fields = models.IntegerField(null=True, verbose_name='比对字段数目') 322 total_fields = models.IntegerField(null=True, verbose_name='比对字段数目')
323 workflow_name = models.SmallIntegerField(null=True, verbose_name="工作流程") 323 workflow_name = models.CharField(max_length=255, verbose_name="工作流程", null=True)
324 324
325 class Meta: 325 class Meta:
326 managed = False 326 managed = False
...@@ -331,15 +331,16 @@ class AFCCompareReport(models.Model): ...@@ -331,15 +331,16 @@ class AFCCompareReport(models.Model):
331 id = models.AutoField(primary_key=True, verbose_name="id") # 主键 331 id = models.AutoField(primary_key=True, verbose_name="id") # 主键
332 case_number = models.CharField(max_length=64, verbose_name="申请id") 332 case_number = models.CharField(max_length=64, verbose_name="申请id")
333 request_team = models.SmallIntegerField(default=RequestTeam.ACCEPTANCE.value, verbose_name="来源") 333 request_team = models.SmallIntegerField(default=RequestTeam.ACCEPTANCE.value, verbose_name="来源")
334 request_trigger = models.SmallIntegerField(default=RequestTrigger.POS.value, verbose_name="触发者") 334 request_trigger = models.SmallIntegerField(default=RequestTrigger.SUBMITING.value, verbose_name="触发者")
335
335 input_file = models.CharField(max_length=255, verbose_name="文件名", null=True) 336 input_file = models.CharField(max_length=255, verbose_name="文件名", null=True)
336 transaction_start = models.DateTimeField(verbose_name='开始时间') # 索引 337 transaction_start = models.DateTimeField(verbose_name='开始时间') # 索引
337 transaction_end = models.DateTimeField(verbose_name='结束时间') 338 transaction_end = models.DateTimeField(verbose_name='结束时间')
338 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功") 339 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功")
339 failure_reason = models.TextField(null=True, verbose_name="失败原因") 340 failure_reason = models.TextField(null=True, verbose_name="失败原因")
340 process_name = models.SmallIntegerField(default=ProcessName.ALL.value, verbose_name="流程名称") 341 process_name = models.SmallIntegerField(default=ProcessName.CACOMPARE.value, verbose_name="流程名称")
341 total_fields = models.IntegerField(null=True, verbose_name='比对字段数目') 342 total_fields = models.IntegerField(null=True, verbose_name='比对字段数目')
342 workflow_name = models.SmallIntegerField(null=True, verbose_name="工作流程") 343 workflow_name = models.CharField(max_length=255, verbose_name="工作流程", null=True)
343 344
344 class Meta: 345 class Meta:
345 managed = False 346 managed = False
...@@ -352,13 +353,13 @@ class HILCompareOfflineReport(models.Model): ...@@ -352,13 +353,13 @@ class HILCompareOfflineReport(models.Model):
352 case_number = models.CharField(max_length=255, verbose_name="申请id") 353 case_number = models.CharField(max_length=255, verbose_name="申请id")
353 request_team = models.CharField(max_length=255, verbose_name="来源") 354 request_team = models.CharField(max_length=255, verbose_name="来源")
354 request_trigger = models.CharField(max_length=1024, verbose_name="触发者") 355 request_trigger = models.CharField(max_length=1024, verbose_name="触发者")
355 input_file = models.CharField(max_length=2048, verbose_name="文件名", null=True) 356 input_file = models.CharField(max_length=2048, verbose_name="文件名")
356 transaction_start = models.DateTimeField(verbose_name='开始时间') # 索引 357 transaction_start = models.DateTimeField(verbose_name='开始时间') # 索引
357 transaction_end = models.DateTimeField(verbose_name='结束时间') 358 transaction_end = models.DateTimeField(verbose_name='结束时间')
358 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功") 359 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功")
359 failure_reason = models.TextField(null=True, verbose_name="失败原因") 360 failure_reason = models.TextField(null=True, verbose_name="失败原因")
360 process_name = models.CharField(max_length=1024, verbose_name="流程名称") 361 process_name = models.CharField(max_length=1024, verbose_name="流程名称")
361 total_fields = models.IntegerField(verbose_name='比对字段数目') 362 total_fields = models.IntegerField(verbose_name='比对字段数目', null=True)
362 workflow_name = models.CharField(max_length=1024, verbose_name="工作流程") 363 workflow_name = models.CharField(max_length=1024, verbose_name="工作流程")
363 364
364 class Meta: 365 class Meta:
...@@ -371,13 +372,13 @@ class AFCCompareOfflineReport(models.Model): ...@@ -371,13 +372,13 @@ class AFCCompareOfflineReport(models.Model):
371 case_number = models.CharField(max_length=255, verbose_name="申请id") 372 case_number = models.CharField(max_length=255, verbose_name="申请id")
372 request_team = models.CharField(max_length=255, verbose_name="来源") 373 request_team = models.CharField(max_length=255, verbose_name="来源")
373 request_trigger = models.CharField(max_length=1024, verbose_name="触发者") 374 request_trigger = models.CharField(max_length=1024, verbose_name="触发者")
374 input_file = models.CharField(max_length=2048, verbose_name="文件名", null=True) 375 input_file = models.CharField(max_length=2048, verbose_name="文件名")
375 transaction_start = models.DateTimeField(verbose_name='开始时间') # 索引 376 transaction_start = models.DateTimeField(verbose_name='开始时间') # 索引
376 transaction_end = models.DateTimeField(verbose_name='结束时间') 377 transaction_end = models.DateTimeField(verbose_name='结束时间')
377 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功") 378 successful_at_this_level = models.BooleanField(default=True, verbose_name="是否成功")
378 failure_reason = models.TextField(null=True, verbose_name="失败原因") 379 failure_reason = models.TextField(null=True, verbose_name="失败原因")
379 process_name = models.CharField(max_length=1024, verbose_name="流程名称") 380 process_name = models.CharField(max_length=1024, verbose_name="流程名称")
380 total_fields = models.IntegerField(verbose_name='比对字段数目') 381 total_fields = models.IntegerField(verbose_name='比对字段数目', null=True)
381 workflow_name = models.CharField(max_length=1024, verbose_name="工作流程") 382 workflow_name = models.CharField(max_length=1024, verbose_name="工作流程")
382 383
383 class Meta: 384 class Meta:
......
src/apps/doc/named_enum.py
...@@ -33,6 +33,8 @@ class RequestTrigger(NamedEnum): ...@@ -33,6 +33,8 @@ class RequestTrigger(NamedEnum):
33 EAPP = (1, 'EAPP') 33 EAPP = (1, 'EAPP')
34 ECONTRACT = (2, 'ECONTRACT') 34 ECONTRACT = (2, 'ECONTRACT')
35 DOCUPLOAD = (3, 'Document Upload') 35 DOCUPLOAD = (3, 'Document Upload')
36 SUBMITING = (4, 'Submiting')
37 UPLOADING = (5, 'Uploading')
36 38
37 39
38 class FailureReason(NamedEnum): 40 class FailureReason(NamedEnum):
...@@ -46,6 +48,7 @@ class ProcessName(NamedEnum): ...@@ -46,6 +48,7 @@ class ProcessName(NamedEnum):
46 BS = (1, 'S1_CA_BankStatementCalculation') 48 BS = (1, 'S1_CA_BankStatementCalculation')
47 IDCARD = (2, 'F2_IDReport') 49 IDCARD = (2, 'F2_IDReport')
48 DDA = (3, 'CL_S1_DDAConsolidation') 50 DDA = (3, 'CL_S1_DDAConsolidation')
51 CACOMPARE = (4, 'S1_CA_DocumentVerify')
49 52
50 53
51 class WorkflowName(NamedEnum): 54 class WorkflowName(NamedEnum):
......
src/celery_compare/tasks.py
...@@ -5,10 +5,11 @@ import traceback ...@@ -5,10 +5,11 @@ import traceback
5 from datetime import datetime, timedelta 5 from datetime import datetime, timedelta
6 from collections import OrderedDict 6 from collections import OrderedDict
7 from . import app 7 from . import app
8 from apps.doc.models import AFCOCRResult, HILOCRResult, AFCComparisonInfo, HILComparisonInfo, Configs 8 from apps.doc.models import AFCOCRResult, HILOCRResult, AFCComparisonInfo, HILComparisonInfo, Configs, HILCompareReport, AFCCompareReport
9 from apps.doc import consts 9 from apps.doc import consts
10 from apps.doc.ocr.gcap import gcap 10 from apps.doc.ocr.gcap import gcap
11 from apps.doc.exceptions import GCAPException 11 from apps.doc.exceptions import GCAPException
12 from apps.doc.named_enum import RequestTeam, RequestTrigger, ProcessName
12 from common.tools.comparison import cp 13 from common.tools.comparison import cp
13 14
14 compare_log = logging.getLogger('compare') 15 compare_log = logging.getLogger('compare')
...@@ -173,6 +174,9 @@ def compare(application_id, application_entity, uniq_seq, ocr_res_id): ...@@ -173,6 +174,9 @@ def compare(application_id, application_entity, uniq_seq, ocr_res_id):
173 )) 174 ))
174 return 175 return
175 176
177 start_time = datetime.now()
178 compare_failed = False
179
176 # 比对信息 180 # 比对信息
177 try: 181 try:
178 comparison_res = OrderedDict({ 182 comparison_res = OrderedDict({
...@@ -182,7 +186,7 @@ def compare(application_id, application_entity, uniq_seq, ocr_res_id): ...@@ -182,7 +186,7 @@ def compare(application_id, application_entity, uniq_seq, ocr_res_id):
182 'applicationEntity': application_entity, 186 'applicationEntity': application_entity,
183 'applicationVersion': last_obj.application_version, 187 'applicationVersion': last_obj.application_version,
184 'vehicleStatus': last_obj.vehicle_status, 188 'vehicleStatus': last_obj.vehicle_status,
185 'wholeResult': 'Y', 189 'wholeResult': 'N',
186 'wholeResultMessage': '', 190 'wholeResultMessage': '',
187 'applicationLink': r'//china.bmw.corp/WINFS/SF-CN-data/SF3-CN-S/SF3-CN-S-1-New Business/CA/OCR II test', 191 'applicationLink': r'//china.bmw.corp/WINFS/SF-CN-data/SF3-CN-S/SF3-CN-S-1-New Business/CA/OCR II test',
188 } 192 }
...@@ -278,6 +282,7 @@ def compare(application_id, application_entity, uniq_seq, ocr_res_id): ...@@ -278,6 +282,7 @@ def compare(application_id, application_entity, uniq_seq, ocr_res_id):
278 comparison_res['OCR_Input']['wholeResult'] = consts.RESULT_N if consts.RESULT_N in res_set or consts.RESULT_NA in res_set else consts.RESULT_Y 282 comparison_res['OCR_Input']['wholeResult'] = consts.RESULT_N if consts.RESULT_N in res_set or consts.RESULT_NA in res_set else consts.RESULT_Y
279 283
280 except Exception as e: 284 except Exception as e:
285 compare_failed = True
281 compare_log.error('{0} [compare error] [entity={1}] [id={2}] [uniq_seq={3}] [ocr_res_id={4}] ' 286 compare_log.error('{0} [compare error] [entity={1}] [id={2}] [uniq_seq={3}] [ocr_res_id={4}] '
282 '[error={5}]'.format(log_base, application_entity, application_id, uniq_seq, ocr_res_id, 287 '[error={5}]'.format(log_base, application_entity, application_id, uniq_seq, ocr_res_id,
283 traceback.format_exc())) 288 traceback.format_exc()))
...@@ -327,3 +332,99 @@ def compare(application_id, application_entity, uniq_seq, ocr_res_id): ...@@ -327,3 +332,99 @@ def compare(application_id, application_entity, uniq_seq, ocr_res_id):
327 ocr_res_id, res_text)) 332 ocr_res_id, res_text))
328 compare_log.info('{0} [task success] [entity={1}] [id={2}] [uniq_seq={3}] [ocr_res_id={4}]'.format( 333 compare_log.info('{0} [task success] [entity={1}] [id={2}] [uniq_seq={3}] [ocr_res_id={4}]'.format(
329 log_base, application_entity, application_id, uniq_seq, ocr_res_id)) 334 log_base, application_entity, application_id, uniq_seq, ocr_res_id))
335 finally:
336 try:
337 end_time = datetime.now()
338 if compare_failed:
339 successful_at_this_level = False
340 failure_reason = 'Compare process error'
341 total_fields = 0
342 else:
343 successful_at_this_level = True if comparison_res['OCR_Input']['wholeResult'] == consts.RESULT_Y else False
344 field_failed = {
345 'individualCusInfo': [],
346 'corporateCusInfo': [],
347 'usedCarInfo': []
348 }
349
350 individual_list = comparison_res.get('OCR_Input', {}).get('individualCusInfo', [])
351 total_fields = len(individual_list) * 4
352
353 for individual in individual_list:
354 field_list = []
355 if not successful_at_this_level:
356 for field_name, _, _, _, result_field in consts.ITPRC:
357 if individual.get(result_field) != consts.RESULT_Y:
358 field_list.append(field_name)
359
360 if individual.get('secondIdType') in consts.SECOND_ID_TYPE_COMPARE:
361 total_fields += 1
362 if not successful_at_this_level:
363 if individual.get(consts.SECOND_ID_RES) != consts.RESULT_Y:
364 field_list.append(consts.SECOND_ID_FIELD)
365
366 if individual.get('customerType') == consts.CUSTOMER_TYPE[5]:
367 total_fields += 3
368 if not successful_at_this_level:
369 for field_name, _, _, _, result_field in consts.TCSEP:
370 if individual.get(result_field) != consts.RESULT_Y:
371 field_list.append(field_name)
372
373 if len(field_list) > 0:
374 field_failed['individualCusInfo'].append(';'.join(field_list))
375
376 corporate_res = comparison_res.get('OCR_Input', {}).get('corporateCusInfo')
377 if corporate_res is not None:
378 total_fields += 8
379 if not successful_at_this_level:
380 corporate_field_list = []
381 for field_name, _, _, _, result_field in consts.TCCOR:
382 if corporate_res.get(result_field) != consts.RESULT_Y:
383 corporate_field_list.append(field_name)
384 if len(corporate_field_list) > 0:
385 field_failed['corporateCusInfo'].append(';'.join(corporate_field_list))
386
387 used_car_res = comparison_res.get('OCR_Input', {}).get('usedCarInfo')
388 if used_car_res is not None:
389 total_fields += 3
390 if not successful_at_this_level:
391 used_car_field_list = []
392 for field_name, _, _, _, result_field in consts.PCUSD_MVC:
393 if used_car_res.get(result_field) != consts.RESULT_Y:
394 used_car_field_list.append(field_name)
395 if len(used_car_field_list) > 0:
396 field_failed['usedCarInfo'].append(';'.join(used_car_field_list))
397
398 if successful_at_this_level:
399 reason_list = []
400 for key, value in field_failed.items():
401 if len(value) > 0:
402 value_str = json.dumps(value)
403 reason_list.append('{0}: {1}'.format(key, value_str))
404 failure_reason = '、'.join(reason_list)
405
406 else:
407 failure_reason = ''
408
409
410 request_trigger = RequestTrigger.SUBMITING.value if ocr_res_id is None else RequestTrigger.UPLOADING.value
411 report_class = HILCompareReport if application_entity == consts.HIL_PREFIX else AFCCompareReport
412 report_class.objects.create(
413 case_number=application_id,
414 request_team=RequestTeam.ACCEPTANCE.value,
415 request_trigger=request_trigger,
416 transaction_start=start_time,
417 transaction_end=end_time,
418 successful_at_this_level=successful_at_this_level,
419 failure_reason=failure_reason,
420 process_name=ProcessName.CACOMPARE.value,
421 total_fields=total_fields,
422 workflow_name=last_obj.customer_type,
423 )
424 compare_log.info('{0} [report save success] [entity={1}] [id={2}] [uniq_seq={3}] [ocr_res_id={4}]'.format(
425 log_base, application_entity, application_id, uniq_seq, ocr_res_id))
426 except Exception as e:
427 compare_log.error('{0} [report save success] [entity={1}] [id={2}] [uniq_seq={3}] [ocr_res_id={4}] '
428 '[error={5}]'.format(log_base, application_entity, application_id, uniq_seq,
429 ocr_res_id, traceback.format_exc()))
430
......
src/common/mssql_script2.py 0 → 100644
1 import pyodbc
2
3 hil_sql = """
4
5 """
6
7 afc_sql = """
8
9 """
10
11 hil_cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};', autocommit=True)
12
13 hil_cursor = hil_cnxn.cursor()
14 hil_cursor.execute(hil_sql)
15
16 hil_cursor.close()
17 hil_cnxn.close()
18
19 afc_cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};', autocommit=True)
20
21 afc_cursor = afc_cnxn.cursor()
22 afc_cursor.execute(afc_sql)
23
24 afc_cursor.close()
25 afc_cnxn.close()
src/common/tools/mssql_script3.py
1 import pyodbc 1 import pyodbc
2 2
3 hil_sql = """ 3 hil_sql = """
4 create table hil_ocr_report
5 (
6 id int identity
7 primary key,
8 case_number nvarchar(64) not null,
9 request_team tinyint default 0 not null,
10 request_trigger tinyint default 0 not null,
11 input_file nvarchar(255) not null,
12 transaction_start datetime not null,
13 transaction_end datetime not null,
14 successful_at_this_level bit default 0 not null,
15 failure_reason tinyint,
16 process_name tinyint default 0 not null,
17 total_fields smallint,
18 workflow_name tinyint
19 );
4 20
21 create index hil_ocr_report_transaction_start_index
22 on hil_ocr_report (transaction_start);
23
24 create table hil_compare_report
25 (
26 id int identity
27 primary key,
28 case_number nvarchar(64) not null,
29 request_team tinyint default 0 not null,
30 request_trigger tinyint default 4 not null,
31 input_file nvarchar(255),
32 transaction_start datetime not null,
33 transaction_end datetime not null,
34 successful_at_this_level bit default 0 not null,
35 failure_reason nvarchar(max),
36 process_name tinyint default 4 not null,
37 total_fields smallint,
38 workflow_name nvarchar(255)
39 );
40
41 create hil_compare_report_transaction_start_index
42 on hil_compare_report (transaction_start);
43
44 create table hil_compare_offline_report
45 (
46 id int identity
47 primary key,
48 case_number nvarchar(255) not null,
49 request_team nvarchar(255) not null,
50 request_trigger nvarchar(1024) not null,
51 input_file nvarchar(2048) not null,
52 transaction_start datetime not null,
53 transaction_end datetime not null,
54 successful_at_this_level bit default 0 not null,
55 failure_reason nvarchar(max),
56 process_name nvarchar(1024) not null,
57 total_fields smallint,
58 workflow_name nvarchar(1024) not null,
59 );
60
61 create index hil_compare_offline_report_transaction_start_index
62 on hil_compare_offline_report (transaction_start);
5 """ 63 """
6 64
7 afc_sql = """ 65 afc_sql = """
66 create table afc_ocr_report
67 (
68 id int identity
69 primary key,
70 case_number nvarchar(64) not null,
71 request_team tinyint default 0 not null,
72 request_trigger tinyint default 0 not null,
73 input_file nvarchar(255) not null,
74 transaction_start datetime not null,
75 transaction_end datetime not null,
76 successful_at_this_level bit default 0 not null,
77 failure_reason tinyint,
78 process_name tinyint default 0 not null,
79 total_fields smallint,
80 workflow_name tinyint
81 );
82
83 create index afc_ocr_report_transaction_start_index
84 on afc_ocr_report (transaction_start);
85
86 create table afc_compare_report
87 (
88 id int identity
89 primary key,
90 case_number nvarchar(64) not null,
91 request_team tinyint default 0 not null,
92 request_trigger tinyint default 4 not null,
93 input_file nvarchar(255),
94 transaction_start datetime not null,
95 transaction_end datetime not null,
96 successful_at_this_level bit default 0 not null,
97 failure_reason nvarchar(max),
98 process_name tinyint default 4 not null,
99 total_fields smallint,
100 workflow_name nvarchar(255)
101 );
102
103 create index afc_compare_report_transaction_start_index
104 on afc_compare_report (transaction_start);
105
106 create table afc_compare_offline_report
107 (
108 id int identity
109 primary key,
110 case_number nvarchar(255) not null,
111 request_team nvarchar(255) not null,
112 request_trigger nvarchar(1024) not null,
113 input_file nvarchar(2048) not null,
114 transaction_start datetime not null,
115 transaction_end datetime not null,
116 successful_at_this_level bit default 0 not null,
117 failure_reason nvarchar(max),
118 process_name nvarchar(1024) not null,
119 total_fields smallint,
120 workflow_name nvarchar(1024) not null,
121 );
8 122
123 create index afc_compare_offline_report_transaction_start_index
124 on afc_compare_offline_report (transaction_start);
9 """ 125 """
10 126
11 hil_cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};', autocommit=True) 127 hil_cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};', autocommit=True)
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!