diff --git a/src/apps/doc/management/commands/bs_count_statistics.py b/src/apps/doc/management/commands/bs_count_statistics.py new file mode 100644 index 0000000..10d68b2 --- /dev/null +++ b/src/apps/doc/management/commands/bs_count_statistics.py @@ -0,0 +1,188 @@ +import re +import os +import ast +import json +import datetime +from openpyxl import Workbook +from django.core.management import BaseCommand +from django.db.models import Avg, Max, Min +from settings import conf +from common.mixins import LoggerMixin +from apps.doc import consts +from apps.doc.models import HILDoc, AFCDoc, DocStatus + + +class Command(BaseCommand, LoggerMixin): + + def __init__(self): + super().__init__() + self.header_1 = ('date', + 'Daily CA doc handling volume', None, None, + 'Daily CA doc handling application volume', + 'Daily bank statement handling volume', None, None, + 'Daily bank statement handling application volume', + 'Daily average processing time (s)', + 'Daily longest processing time (s)', + 'Daily shortest processing time (s)') + self.header_2 = (None, + 'success', 'failed', 'total', + None, + 'success', 'failed', 'total', + None, + None, + None, + None) + + def add_arguments(self, parser): + parser.add_argument( + '--end_date', + default=datetime.date.today() - datetime.timedelta(days=1), + dest='end_date', + help='将要计算的终止日期,格式: 2018-01-01' + ) + parser.add_argument( + '--start_date', + required=True, + dest='start_date', + help='将要计算的起始日期,格式: 2018-01-01' + ) + + @staticmethod + def gen_dates(start_date, days): + day = datetime.timedelta(days=1) + for i in range(days): + yield start_date + day * i + + def handle(self, *args, **kwargs): + end_date = kwargs.get('end_date') + if isinstance(end_date, str): + if not re.match(r'\d{4}-\d{2}-\d{2}', end_date): + print('date format error') + return + end_date_str = end_date + else: + end_date_str = end_date.strftime('%Y-%m-%d') + + start_date = kwargs.get('start_date') + if isinstance(start_date, str): + if not re.match(r'\d{4}-\d{2}-\d{2}', start_date): + print('date format error') + return + start_date_str = start_date + else: + start_date_str = start_date.strftime('%Y-%m-%d') + + end_date = datetime.datetime.strptime(end_date_str, '%Y-%m-%d') + start_date = datetime.datetime.strptime(start_date_str, '%Y-%m-%d') + days = (end_date - start_date).days + if days < 0: + print('start_date must leq end_date') + return + + excel_dir = os.path.join(conf.DATA_DIR, 'AFC', 'Logs') + if not os.path.exists(excel_dir): + print('excel dir not exists') + return + + wb = Workbook() + ws = wb.get_sheet_by_name('Sheet') + ws.title = 'CA BS' + ws.append(self.header_1) + ws.append(self.header_2) + ws.merge_cells('A1:A2') + ws.merge_cells('E1:E2') + ws.merge_cells('I1:I2') + ws.merge_cells('J1:J2') + ws.merge_cells('K1:K2') + ws.merge_cells('L1:L2') + ws.merge_cells('B1:D1') + ws.merge_cells('F1:H1') + + for date in self.gen_dates(start_date, days): + date_str = date.strftime('%Y-%m-%d') + next_date = date + datetime.timedelta(days=1) + + hil_doc_total = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + document_scheme=consts.DOC_SCHEME_LIST[0]).count() + hil_app_total = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + document_scheme=consts.DOC_SCHEME_LIST[0]).values_list( + 'application_id', flat=True).distinct().count() + hil_app_not_bs = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + document_scheme=consts.DOC_SCHEME_LIST[0], bs_count=0).values_list( + 'application_id', flat=True).distinct().count() + hil_doc_success = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + status=DocStatus.COMPLETE.value, + document_scheme=consts.DOC_SCHEME_LIST[0]).count() + hil_doc_duration = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + status=DocStatus.COMPLETE.value, + document_scheme=consts.DOC_SCHEME_LIST[0]).aggregate( + Avg("duration"), Max("duration"), Min("duration")) + + + afc_doc_total = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + document_scheme=consts.DOC_SCHEME_LIST[0]).count() + afc_app_total = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + document_scheme=consts.DOC_SCHEME_LIST[0]).values_list( + 'application_id', flat=True).distinct().count() + afc_app_not_bs = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + document_scheme=consts.DOC_SCHEME_LIST[0], bs_count=0).values_list( + 'application_id', flat=True).distinct().count() + afc_doc_success = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + status=DocStatus.COMPLETE.value, + document_scheme=consts.DOC_SCHEME_LIST[0]).count() + afc_doc_duration = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date, + status=DocStatus.COMPLETE.value, + document_scheme=consts.DOC_SCHEME_LIST[0]).aggregate( + Avg("duration"), Max("duration"), Min("duration")) + + doc_total = hil_doc_total + afc_doc_total + doc_success = hil_doc_success + afc_doc_success + doc_failed = doc_total - doc_success + + app_total = hil_app_total + afc_app_total + + app_bs = app_total - hil_app_not_bs - afc_app_not_bs + + duration_min = min(hil_doc_duration.get('duration__min'), afc_doc_duration.get('duration__min')) + duration_max = max(hil_doc_duration.get('duration__max'), afc_doc_duration.get('duration__max')) + duration_avg = (hil_doc_duration.get('duration__avg') / (1 + (hil_doc_success / afc_doc_success))) + \ + (afc_doc_duration.get('duration__avg') / (1 + (afc_doc_success / hil_doc_success))) + + success_doc_count = 0 + success_bs_count = 0 + + log_path = os.path.join(conf.LOG_DIR, 'bs.log.{0}'.format(date_str)) + # log_path = os.path.join(conf.LOG_DIR, 'bs_statistics.log.{0}'.format(date_str)) + if not os.path.exists(log_path): + print('log_path not exists') + else: + with open(log_path, 'r', encoding='utf-8') as fp: + for line in fp: + search_obj = re.search(r'task=(.*) merged_bs_summary=(.*)', line) + # search_obj = re.search(r'\[task=(.*)] \[bs_summary=(.*)]', line) + task_str = search_obj.group(1) + business_type, doc_id_str = task_str.split(consts.SPLIT_STR) + doc_id = int(doc_id_str) + doc_class = HILDoc if business_type == consts.HIL_PREFIX else AFCDoc + doc = doc_class.objects.filter(id=doc_id, status=DocStatus.COMPLETE.value, + document_scheme=consts.DOC_SCHEME_LIST[0]).first() + if doc is None: + continue + + bs_summary_str = search_obj.group(2) + new_bs_summary_str = re.sub(r'datetime.date\(\d+, \d+, \d+\)', 'None', bs_summary_str) + bs_summary = ast.literal_eval(new_bs_summary_str) + bs_count = len(bs_summary) + if bs_count > 0: + success_doc_count += 1 + success_bs_count += bs_count + + ws.append((date_str, + doc_success, doc_failed, doc_total, + app_total, + '{0}/{1}'.format(success_doc_count, success_bs_count), None, None, # 只能记录成功的,失败的无法知道是否含有银行流水 + app_bs, # 只能记录成功的,失败的无法知道是否含有银行流水 + duration_avg, duration_max, duration_min)) + + excel_path = os.path.join(excel_dir, 'bs_count_{0}_to_{1}.xlsx'.format(start_date_str, end_date_str)) + wb.save(excel_path) diff --git a/src/apps/doc/management/commands/folder_ocr_process.py b/src/apps/doc/management/commands/folder_ocr_process.py index 14fb633..d7056f4 100644 --- a/src/apps/doc/management/commands/folder_ocr_process.py +++ b/src/apps/doc/management/commands/folder_ocr_process.py @@ -141,6 +141,7 @@ class Command(BaseCommand, LoggerMixin): wb = BSWorkbook(set(), set(), set(), set()) wb.simple_license_rebuild(license_summary, consts.DOC_SCHEME_LIST[0]) + wb.remove_base_sheet() wb.save(excel_path) except Exception as e: self.folder_log.error('{0} [wb build error] [path={1}] [error={2}]'.format( diff --git a/src/apps/doc/management/commands/idcard_daily.py b/src/apps/doc/management/commands/idcard_daily.py index 21ac624..2707327 100644 --- a/src/apps/doc/management/commands/idcard_daily.py +++ b/src/apps/doc/management/commands/idcard_daily.py @@ -35,8 +35,10 @@ class Command(BaseCommand, LoggerMixin): else: date_str = date.strftime('%Y-%m-%d') - afc_excel_dir = os.path.join(conf.DATA_DIR, 'AFC', 'IdCard') - hil_excel_dir = os.path.join(conf.DATA_DIR, 'HIL', 'IdCard') + # afc_excel_dir = os.path.join(conf.DATA_DIR, 'AFC', 'IdCard') + # hil_excel_dir = os.path.join(conf.DATA_DIR, 'HIL', 'IdCard') + afc_excel_dir = conf.IC_REPORT_AFC + hil_excel_dir = conf.IC_REPORT_HIL if not os.path.exists(afc_excel_dir) or not os.path.exists(hil_excel_dir): print('excel_dir not exist') return diff --git a/src/apps/doc/management/commands/idcard_monthly.py b/src/apps/doc/management/commands/idcard_monthly.py index 2a99dc5..244ac01 100644 --- a/src/apps/doc/management/commands/idcard_monthly.py +++ b/src/apps/doc/management/commands/idcard_monthly.py @@ -19,7 +19,10 @@ class Command(BaseCommand, LoggerMixin): pre_mouth = end_day_in_mouth - datetime.timedelta(days=1) for target_dir in self.dirs: - excel_dir = os.path.join(conf.DATA_DIR, target_dir, 'IdCard') + if target_dir == 'AFC': + excel_dir = conf.IC_REPORT_AFC + else: + excel_dir = conf.IC_REPORT_HIL if not os.path.exists(excel_dir): print('excel dir not exists: {0}'.format(excel_dir)) return