Skip to content

周伟奇 / bmw-ocr

Go to a project
Toggle navigation
Toggle navigation pinning
1762216b by 周伟奇
Options

add bs count statistics script

1 parent 5e463cbd
Inline Side-by-side
Showing 1 changed file with 188 additions and 0 deletions
src/apps/doc/management/commands/bs_count_statistics.py 0 → 100644
1 import re
2 import os
3 import ast
4 import json
5 import datetime
6 from openpyxl import Workbook
7 from django.core.management import BaseCommand
8 from django.db.models import Avg, Max, Min
9 from settings import conf
10 from common.mixins import LoggerMixin
11 from apps.doc import consts
12 from apps.doc.models import HILDoc, AFCDoc, DocStatus
13
14
15 class Command(BaseCommand, LoggerMixin):
16
17 def __init__(self):
18 super().__init__()
19 self.header_1 = ('date',
20 'Daily CA doc handling volume', None, None,
21 'Daily CA doc handling application volume',
22 'Daily bank statement handling volume', None, None,
23 'Daily bank statement handling application volume',
24 'Daily average processing time (s)',
25 'Daily longest processing time (s)',
26 'Daily shortest processing time (s)')
27 self.header_2 = (None,
28 'success', 'failed', 'total',
29 None,
30 'success', 'failed', 'total',
31 None,
32 None,
33 None,
34 None)
35
36 def add_arguments(self, parser):
37 parser.add_argument(
38 '--end_date',
39 default=datetime.date.today() - datetime.timedelta(days=1),
40 dest='end_date',
41 help='将要计算的终止日期，格式: 2018-01-01'
42 )
43 parser.add_argument(
44 '--start_date',
45 required=True,
46 dest='start_date',
47 help='将要计算的起始日期，格式: 2018-01-01'
48 )
49
50 @staticmethod
51 def gen_dates(start_date, days):
52 day = datetime.timedelta(days=1)
53 for i in range(days):
54 yield start_date + day * i
55
56 def handle(self, *args, **kwargs):
57 end_date = kwargs.get('end_date')
58 if isinstance(end_date, str):
59 if not re.match(r'\d{4}-\d{2}-\d{2}', end_date):
60 print('date format error')
61 return
62 end_date_str = end_date
63 else:
64 end_date_str = end_date.strftime('%Y-%m-%d')
65
66 start_date = kwargs.get('start_date')
67 if isinstance(start_date, str):
68 if not re.match(r'\d{4}-\d{2}-\d{2}', start_date):
69 print('date format error')
70 return
71 start_date_str = start_date
72 else:
73 start_date_str = start_date.strftime('%Y-%m-%d')
74
75 end_date = datetime.datetime.strptime(end_date_str, '%Y-%m-%d')
76 start_date = datetime.datetime.strptime(start_date_str, '%Y-%m-%d')
77 days = (end_date - start_date).days
78 if days < 0:
79 print('start_date must leq end_date')
80 return
81
82 excel_dir = os.path.join(conf.DATA_DIR, 'AFC', 'Logs')
83 if not os.path.exists(excel_dir):
84 print('excel dir not exists')
85 return
86
87 wb = Workbook()
88 ws = wb.get_sheet_by_name('Sheet')
89 ws.title = 'CA BS'
90 ws.append(self.header_1)
91 ws.append(self.header_2)
92 ws.merge_cells('A1:A2')
93 ws.merge_cells('E1:E2')
94 ws.merge_cells('I1:I2')
95 ws.merge_cells('J1:J2')
96 ws.merge_cells('K1:K2')
97 ws.merge_cells('L1:L2')
98 ws.merge_cells('B1:D1')
99 ws.merge_cells('F1:H1')
100
101 for date in self.gen_dates(start_date, days):
102 date_str = date.strftime('%Y-%m-%d')
103 next_date = date + datetime.timedelta(days=1)
104
105 hil_doc_total = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
106 document_scheme=consts.DOC_SCHEME_LIST[0]).count()
107 hil_app_total = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
108 document_scheme=consts.DOC_SCHEME_LIST[0]).values_list(
109 'application_id', flat=True).distinct().count()
110 hil_app_not_bs = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
111 document_scheme=consts.DOC_SCHEME_LIST[0], bs_count=0).values_list(
112 'application_id', flat=True).distinct().count()
113 hil_doc_success = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
114 status=DocStatus.COMPLETE.value,
115 document_scheme=consts.DOC_SCHEME_LIST[0]).count()
116 hil_doc_duration = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
117 status=DocStatus.COMPLETE.value,
118 document_scheme=consts.DOC_SCHEME_LIST[0]).aggregate(
119 Avg("duration"), Max("duration"), Min("duration"))
120
121
122 afc_doc_total = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
123 document_scheme=consts.DOC_SCHEME_LIST[0]).count()
124 afc_app_total = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
125 document_scheme=consts.DOC_SCHEME_LIST[0]).values_list(
126 'application_id', flat=True).distinct().count()
127 afc_app_not_bs = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
128 document_scheme=consts.DOC_SCHEME_LIST[0], bs_count=0).values_list(
129 'application_id', flat=True).distinct().count()
130 afc_doc_success = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
131 status=DocStatus.COMPLETE.value,
132 document_scheme=consts.DOC_SCHEME_LIST[0]).count()
133 afc_doc_duration = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
134 status=DocStatus.COMPLETE.value,
135 document_scheme=consts.DOC_SCHEME_LIST[0]).aggregate(
136 Avg("duration"), Max("duration"), Min("duration"))
137
138 doc_total = hil_doc_total + afc_doc_total
139 doc_success = hil_doc_success + afc_doc_success
140 doc_failed = doc_total - doc_success
141
142 app_total = hil_app_total + afc_app_total
143
144 app_bs = app_total - hil_app_not_bs - afc_app_not_bs
145
146 duration_min = min(hil_doc_duration.get('duration__min'), afc_doc_duration.get('duration__min'))
147 duration_max = max(hil_doc_duration.get('duration__max'), afc_doc_duration.get('duration__max'))
148 duration_avg = (hil_doc_duration.get('duration__avg') / (1 + (hil_doc_success / afc_doc_success))) + \
149 (afc_doc_duration.get('duration__avg') / (1 + (afc_doc_success / hil_doc_success)))
150
151 success_doc_count = 0
152 success_bs_count = 0
153
154 log_path = os.path.join(conf.LOG_DIR, 'bs.log.{0}'.format(date_str))
155 # log_path = os.path.join(conf.LOG_DIR, 'bs_statistics.log.{0}'.format(date_str))
156 if not os.path.exists(log_path):
157 print('log_path not exists')
158 else:
159 with open(log_path, 'r', encoding='utf-8') as fp:
160 for line in fp:
161 search_obj = re.search(r'task=(.*) merged_bs_summary=(.*)', line)
162 # search_obj = re.search(r'\[task=(.*)] \[bs_summary=(.*)]', line)
163 task_str = search_obj.group(1)
164 business_type, doc_id_str = task_str.split(consts.SPLIT_STR)
165 doc_id = int(doc_id_str)
166 doc_class = HILDoc if business_type == consts.HIL_PREFIX else AFCDoc
167 doc = doc_class.objects.filter(id=doc_id, status=DocStatus.COMPLETE.value,
168 document_scheme=consts.DOC_SCHEME_LIST[0]).first()
169 if doc is None:
170 continue
171
172 bs_summary_str = search_obj.group(2)
173 new_bs_summary_str = re.sub(r'datetime.date\(\d+, \d+, \d+\)', 'None', bs_summary_str)
174 bs_summary = ast.literal_eval(new_bs_summary_str)
175 bs_count = len(bs_summary)
176 if bs_count > 0:
177 success_doc_count += 1
178 success_bs_count += bs_count
179
180 ws.append((date_str,
181 doc_success, doc_failed, doc_total,
182 app_total,
183 '{0}/{1}'.format(success_doc_count, success_bs_count), None, None, # 只能记录成功的，失败的无法知道是否含有银行流水
184 app_bs, # 只能记录成功的，失败的无法知道是否含有银行流水
185 duration_avg, duration_max, duration_min))
186
187 excel_path = os.path.join(excel_dir, 'bs_count_{0}_to_{1}.xlsx'.format(start_date_str, end_date_str))
188 wb.save(excel_path)
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!