add bs count statistics script

+import re
+import os
+import ast
+import json
+import datetime
+from openpyxl import Workbook
+from django.core.management import BaseCommand
+from django.db.models import Avg, Max, Min
+from settings import conf
+from common.mixins import LoggerMixin
+from apps.doc import consts
+from apps.doc.models import HILDoc, AFCDoc, DocStatus
+
+
+class Command(BaseCommand, LoggerMixin):
+
+    def __init__(self):
+        super().__init__()
+        self.header_1 = ('date',
+                         'Daily CA doc handling volume', None, None,
+                         'Daily CA doc handling application volume',
+                         'Daily bank statement handling volume', None, None,
+                         'Daily  bank statement handling application volume',
+                         'Daily average processing time (s)',
+                         'Daily longest processing time (s)',
+                         'Daily shortest processing time (s)')
+        self.header_2 = (None,
+                         'success', 'failed', 'total',
+                         None,
+                         'success', 'failed', 'total',
+                         None,
+                         None,
+                         None,
+                         None)
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--end_date',
+            default=datetime.date.today() - datetime.timedelta(days=1),
+            dest='end_date',
+            help='将要计算的终止日期，格式: 2018-01-01'
+        )
+        parser.add_argument(
+            '--start_date',
+            required=True,
+            dest='start_date',
+            help='将要计算的起始日期，格式: 2018-01-01'
+        )
+
+    @staticmethod
+    def gen_dates(start_date, days):
+        day = datetime.timedelta(days=1)
+        for i in range(days):
+            yield start_date + day * i
+
+    def handle(self, *args, **kwargs):
+        end_date = kwargs.get('end_date')
+        if isinstance(end_date, str):
+            if not re.match(r'\d{4}-\d{2}-\d{2}', end_date):
+                print('date format error')
+                return
+            end_date_str = end_date
+        else:
+            end_date_str = end_date.strftime('%Y-%m-%d')
+
+        start_date = kwargs.get('start_date')
+        if isinstance(start_date, str):
+            if not re.match(r'\d{4}-\d{2}-\d{2}', start_date):
+                print('date format error')
+                return
+            start_date_str = start_date
+        else:
+            start_date_str = start_date.strftime('%Y-%m-%d')
+
+        end_date = datetime.datetime.strptime(end_date_str, '%Y-%m-%d')
+        start_date = datetime.datetime.strptime(start_date_str, '%Y-%m-%d')
+        days = (end_date - start_date).days
+        if days < 0:
+            print('start_date must leq end_date')
+            return
+
+        excel_dir = os.path.join(conf.DATA_DIR, 'AFC', 'Logs')
+        if not os.path.exists(excel_dir):
+            print('excel dir not exists')
+            return
+
+        wb = Workbook()
+        ws = wb.get_sheet_by_name('Sheet')
+        ws.title = 'CA BS'
+        ws.append(self.header_1)
+        ws.append(self.header_2)
+        ws.merge_cells('A1:A2')
+        ws.merge_cells('E1:E2')
+        ws.merge_cells('I1:I2')
+        ws.merge_cells('J1:J2')
+        ws.merge_cells('K1:K2')
+        ws.merge_cells('L1:L2')
+        ws.merge_cells('B1:D1')
+        ws.merge_cells('F1:H1')
+
+        for date in self.gen_dates(start_date, days):
+            date_str = date.strftime('%Y-%m-%d')
+            next_date = date + datetime.timedelta(days=1)
+
+            hil_doc_total = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                  document_scheme=consts.DOC_SCHEME_LIST[0]).count()
+            hil_app_total = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                  document_scheme=consts.DOC_SCHEME_LIST[0]).values_list(
+                'application_id', flat=True).distinct().count()
+            hil_app_not_bs = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                   document_scheme=consts.DOC_SCHEME_LIST[0], bs_count=0).values_list(
+                'application_id', flat=True).distinct().count()
+            hil_doc_success = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                    status=DocStatus.COMPLETE.value,
+                                                    document_scheme=consts.DOC_SCHEME_LIST[0]).count()
+            hil_doc_duration = HILDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                     status=DocStatus.COMPLETE.value,
+                                                     document_scheme=consts.DOC_SCHEME_LIST[0]).aggregate(
+                Avg("duration"), Max("duration"), Min("duration"))
+
+
+            afc_doc_total = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                  document_scheme=consts.DOC_SCHEME_LIST[0]).count()
+            afc_app_total = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                  document_scheme=consts.DOC_SCHEME_LIST[0]).values_list(
+                'application_id', flat=True).distinct().count()
+            afc_app_not_bs = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                   document_scheme=consts.DOC_SCHEME_LIST[0], bs_count=0).values_list(
+                'application_id', flat=True).distinct().count()
+            afc_doc_success = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                    status=DocStatus.COMPLETE.value,
+                                                    document_scheme=consts.DOC_SCHEME_LIST[0]).count()
+            afc_doc_duration = AFCDoc.objects.filter(create_time__gt=date, create_time__lt=next_date,
+                                                     status=DocStatus.COMPLETE.value,
+                                                     document_scheme=consts.DOC_SCHEME_LIST[0]).aggregate(
+                Avg("duration"), Max("duration"), Min("duration"))
+
+            doc_total = hil_doc_total + afc_doc_total
+            doc_success = hil_doc_success + afc_doc_success
+            doc_failed = doc_total - doc_success
+
+            app_total = hil_app_total + afc_app_total
+
+            app_bs = app_total - hil_app_not_bs - afc_app_not_bs
+
+            duration_min = min(hil_doc_duration.get('duration__min'), afc_doc_duration.get('duration__min'))
+            duration_max = max(hil_doc_duration.get('duration__max'), afc_doc_duration.get('duration__max'))
+            duration_avg = (hil_doc_duration.get('duration__avg') / (1 + (hil_doc_success / afc_doc_success))) + \
+                           (afc_doc_duration.get('duration__avg') / (1 + (afc_doc_success / hil_doc_success)))
+
+            success_doc_count = 0
+            success_bs_count = 0
+
+            log_path = os.path.join(conf.LOG_DIR, 'bs.log.{0}'.format(date_str))
+            # log_path = os.path.join(conf.LOG_DIR, 'bs_statistics.log.{0}'.format(date_str))
+            if not os.path.exists(log_path):
+                print('log_path not exists')
+            else:
+                with open(log_path, 'r', encoding='utf-8') as fp:
+                    for line in fp:
+                        search_obj = re.search(r'task=(.*) merged_bs_summary=(.*)', line)
+                        # search_obj = re.search(r'\[task=(.*)] \[bs_summary=(.*)]', line)
+                        task_str = search_obj.group(1)
+                        business_type, doc_id_str = task_str.split(consts.SPLIT_STR)
+                        doc_id = int(doc_id_str)
+                        doc_class = HILDoc if business_type == consts.HIL_PREFIX else AFCDoc
+                        doc = doc_class.objects.filter(id=doc_id, status=DocStatus.COMPLETE.value,
+                                                       document_scheme=consts.DOC_SCHEME_LIST[0]).first()
+                        if doc is None:
+                            continue
+
+                        bs_summary_str = search_obj.group(2)
+                        new_bs_summary_str = re.sub(r'datetime.date\(\d+, \d+, \d+\)', 'None', bs_summary_str)
+                        bs_summary = ast.literal_eval(new_bs_summary_str)
+                        bs_count = len(bs_summary)
+                        if bs_count > 0:
+                            success_doc_count += 1
+                            success_bs_count += bs_count
+
+            ws.append((date_str,
+                       doc_success, doc_failed, doc_total,
+                       app_total,
+                       '{0}/{1}'.format(success_doc_count, success_bs_count), None, None,  # 只能记录成功的，失败的无法知道是否含有银行流水
+                       app_bs,  # 只能记录成功的，失败的无法知道是否含有银行流水
+                       duration_avg, duration_max, duration_min))
+
+        excel_path = os.path.join(excel_dir, 'bs_count_{0}_to_{1}.xlsx'.format(start_date_str, end_date_str))
+        wb.save(excel_path)