import os
import re
import time
import shutil
import base64
import signal
import requests
import traceback
from django import db
from PIL import Image
from datetime import datetime
from django.core.management import BaseCommand
from multiprocessing import Process
import numpy as np
from fuzzywuzzy import fuzz
from shapely.geometry import Polygon
from settings import conf
from common.mixins import LoggerMixin
from common.tools.pdf_to_img import PDFHandler
from apps.doc import consts
from apps.doc.exceptions import OCR1Exception, OCR4Exception
from apps.doc.ocr.wb import BSWorkbook, PatternFill
from apps.doc.models import OfflineReport, AFCOfflineReport
from apps.doc.named_enum import OfflineFailureReason
class Finder:
"""Summary
Attributes:
ocr_results (TYPE): Description
"""
def __init__(self, ocr_results=None):
self.ocr_results = ocr_results
self.init_result = {
"合同编号列表": [],
"经销商名称_Page3": "",
"经销商名称_Page38": "",
"经销商统一社会信用代码或公司注册号": "",
"保证人": "",
"综合授信额度金额英文": "",
"综合授信额度金额中文": "",
"综合授信额度期限开始日期英文": '',
"综合授信额度期限截止日期英文": '',
"综合授信额度期限开始日期中文": '',
"综合授信额度期限截止日期中文": '',
"保证金比例英文": "",
"保证金比例中文": "",
"其他约定与条件英文": "",
"其他约定与条件中文": "",
}
def get_line(self, ocr_results, key_string):
# 根据指定关键词, 找出与关键词同处一行的字符
top, bottom = -1, -1
# 首先找到这个关键词所在的 Bbox
for key in ocr_results:
bbox, text = ocr_results[key]
if key_string in text:
top, bottom = min(bbox[1::2]), max(bbox[1::2])
break
line_text = []
# 然后找到一行
for key in ocr_results:
bbox, text = ocr_results[key]
if top < np.mean(bbox[1::2]) < bottom:
line_text.append([bbox, text])
# 从左到右排序
lines = ''
if len(line_text) > 0:
line_text = sorted(line_text, key=lambda x: x[0][0], reverse=False)
lines = ''.join([i[1] for i in line_text])
return lines
def page_predict(self, ocr_results, page_template):
classes = []
for pno in ocr_results:
ocr_texts = ''
for key in ocr_results[pno]:
bbox, text = ocr_results[pno][key]
ocr_texts += text
pattern = re.compile("[^\u4e00-\u9fa5]") # 匹配不是中文的其他字符
ocr_texts = pattern.sub('', ocr_texts)
score = fuzz.ratio(page_template, ocr_texts)/100.
classes.append([pno, score])
pred = sorted(classes, key=lambda x: x[1], reverse=True)[0]
return pred
def get_top_key(self, ocr_results, key_string): # 加入过滤词典
"""找到与 key_string 最匹配的字段的 key
"""
if len(ocr_results) == 0:
return -1, -1
ratio_list = [[fuzz.ratio(key_string, ocr_results[key][1]), key] for key in ocr_results]
top_key = sorted(ratio_list, key=lambda x: x[0])[-1]
return top_key
def get_top_iou(self, ocr_results, poly):
"""求最大IoU
"""
iou_list = []
for key in ocr_results:
bbox, text = ocr_results[key]
g = Polygon(np.array(bbox).reshape((-1, 2)))
p = Polygon(np.array(poly).reshape((-1, 2)))
if not g.is_valid or not p.is_valid:
continue
inter = Polygon(g).intersection(Polygon(p)).area
union = g.area + p.area - inter
iou = inter/union
iou_list.append([iou, key])
if len(iou_list) == 0:
return -1, -1
top_iou = sorted(iou_list, key=lambda x: x[0])[-1]
return top_iou
def get_key_value(self, ocr_results, key_string):
"""根据 key 查找 value
"""
value = ''
tmp_ocr_results = {}
for key in ocr_results:
bbox, text = ocr_results[key]
# 定制化规则, 比如过滤一些词呀什么的
# 该例中, 我们要去掉非中文字符
pattern = re.compile("[^\u4e00-\u9fa5]") # 匹配不是中文的其他字符
text = pattern.sub('', text)
tmp_ocr_results[key] = [bbox, text]
# 先根据 key_string 找到 key 的位置所在, 再判断该位置是否包含 value
# 若不包含 value, 则往右边一个单位查找 value
ratio, key = self.get_top_key(tmp_ocr_results, key_string)
if ratio > 50:
bbox, text = ocr_results[key]
words = text.strip(key_string).split(':')[-1]
if len(words) == 0:
# 将 bbox 右移一个单位
x0, y0, x1, y1, x2, y2, x3, y3 = bbox
rw = abs(x0-x1)
anchor = [x0+rw, y0, x1+rw, y1, x2+rw, y2, x3+rw, y3]
iou, key = self.get_top_iou(ocr_results, anchor)
if ratio > 0.3:
bbox, text = ocr_results[key]
words = text.strip(key_string).split(':')[-1]
value = words
else:
value = words
return value
def get_contract_No(self):
"""提取左上角的合同编号字段
"""
contract_No_list = []
for pno in self.ocr_results:
# 请务必保证 OCR 结果不为空, 否则直接返回空屁
if len(self.ocr_results[pno]) > 0:
contract_No = self.get_key_value(self.ocr_results[pno], '合同编号')
else:
contract_No = ''
# 临时解决 S 识别成 8 的问题
# TODO！！！
contract_No_list.append(contract_No)
return contract_No_list
def get_info_in_page_3(self):
"""提取第三页上的经销商名称,和经销商统一社会信用代码或公司注册号
"""
dealer_name = ''
dealer_No = ''
template = r"""合同编号宝马汽车金融中国有限公司甲方宝马汽车金融中国有限公司地址中国北京市朝阳区东三环北路霞光里号佳程
广场座层乙方统一社会信用代码或公司注册号地址鉴于甲方是一家依照中国法律合法组建和存续的汽车金融公司愿意
为宝马中国汽车贸易有限公司以下简称宝马中国及华晨宝马汽车有限公司以下简称华晨宝马在中国大陆的宝马集团经
销商提供汽车批售融资服务乙方是一家依据中国法律合法组建和存续与宝马中国和或华晨宝马签署了授权销售合同具
有专营进口和或国产宝马集团产品合法资格的企业本着自愿平等互惠互利的原则甲乙双方经充分协商签署本综合授信
额度合同本合同达成如下条款综合授信额度合同版本""".replace(" ", "").replace("\n", "")
# 首先找到第三页纸, 我们阈值设为0.5
pno, score = self.page_predict(self.ocr_results, template)
if score > 0.5:
if len(self.ocr_results[pno]) > 0:
# print(self.ocr_results[pno])
# 在所有字段中搜索乙方
for key in self.ocr_results[pno]:
bbox, text = self.ocr_results[pno][key]
if '乙方:' in text:
words = text.split(':')[-1].replace('【', '[').replace('{', '[').replace('】', ']')
dealer_name = words
words = self.get_key_value(self.ocr_results[pno], '统一社会信用代码或公司注册号')
dealer_No = words.replace('O', '0')
return dealer_name, dealer_No
def get_info_in_page_38(self):
"""提取第38页上的经销商名称
"""
dealer_name = ''
template = r"""宝马汽车金融中国有限公司合同编号签署页甲方宝马汽车金融中国有限公司盖章姓名姓名职务职务日期乙方汽车销售服务
有限公司盖章姓名姓名职务职务日期综合授信额度合同版本""".replace(" ", "").replace("\n", "")
# 首先找到第38页纸, 我们阈值设为0.5
pno, score = self.page_predict(self.ocr_results, template)
if score > 0.5:
if len(self.ocr_results[pno]) > 0:
for key in self.ocr_results[pno]:
bbox, text = self.ocr_results[pno][key]
if '乙方:' in text:
words = text.split(':')[-1].replace('【', '[').replace('{', '[')
words = re.sub(r'[(（）)盖章《]', "", words)
dealer_name = words
return dealer_name
def get_guarantor(self):
"""提取第10页上保证人段落,所见即所得
"""
guarantor = '[/]'
all_texts = ''
for pno in self.ocr_results:
for key in self.ocr_results[pno]:
bbox, text = self.ocr_results[pno][key]
all_texts += text
searchObj = re.search( r'保证人\[(.*?)\]与甲方', all_texts)
if searchObj:
words = f'[{searchObj.group(1)}]'
words = words.replace('【', '[').replace('】', ']').replace(',', '，').replace('(', '（').replace(')', '）')
guarantor = words
return guarantor
def get_info_in_page_39(self):
"""提取综合授信合同上的一些字段
"""
# Amount of General Credit Line
amount_eng = ''
amount_chn = ''
term_start_eng = ''
term_end_eng = ''
term_start_chn = ''
term_end_chn = ''
deposit_eng = ''
deposit_chn = ''
template = r"""合同编号中国有限公司宝马汽车金融综合授信额度合同附件确认函综合授信额度金额本合同项下的综合授信额度为人民币
大写综合授信额度下面各个业务的授信额度将由甲方以授信额度通知函的方式时不时的通知乙方本合同项下的综合授信额
度可以由甲方根据乙方的信用和财务状况自行决定随时调整本合同项下的综合授信额度应为在本确认函第条的期间内双方
在确认函中确认的授信额度以及甲方向乙方时不时通过书面授信额度通知函以及临时授信额度通知函中沟通的额度的总和
综合授信额度期限从至或者由甲方向乙方通过书面形式在授信额度通知函中沟通的更短期间保证金甲方对乙方的最低保证
金要求为综合授信额度的实际执行的保证金比例以甲方不时另行书面通知根据最新的经销商融资或保证金相关政策或活动
为准综合授信额度合同版本""".replace(" ", "").replace("\n", "")
# 首先找到综合授信合同第一面, 我们阈值设为0.5
pno, score = self.page_predict(self.ocr_results, template)
if score > 0.5:
if len(self.ocr_results[pno]) > 0:
# 根据关键词，找这一行字符
lines = ''
for i in ['RMB', 'CNY']:
lines += self.get_line(self.ocr_results[pno], i)
# searchObj = re.search( r'RMB(.*?)in', lines)
searchObj = re.search(r'[0-9,.]+', lines)
if searchObj:
words = searchObj.group()
amount_eng = words
lines = self.get_line(self.ocr_results[pno], '人民币')
searchObj = re.search( r'大写(.*?)综合', lines)
if searchObj:
words = searchObj.group(1)
pattern = re.compile("[^\u4e00-\u9fa5]") # 匹配不是中文的其他字符
words = pattern.sub('', words)
words = words.replace("仔", "仟").replace("任", "仟")
words = words.replace("值", "佰")
words = words.replace("拐", "捌")
words = words.replace("查", "壹")
words = words.replace("政", "玖")
words = words.replace("垒", "叁")
amount_chn = words
lines = self.get_line(self.ocr_results[pno], 'ending')
if len(lines) > 0:
start, end = lines.split('ending')
searchStart = re.search( r'[0-9]+-[0-9a-zA-Z]+-[0-9]{4}', start)
if searchStart:
words = searchStart.group()
term_start_eng = words
searchEnd = re.search( r'[0-9]+-[0-9a-zA-Z]+-[0-9]{4}', end)
if searchEnd:
words = searchEnd.group()
term_end_eng = words
lines = self.get_line(self.ocr_results[pno], '至')
if len(lines) > 0:
start, end = lines.split('至')
searchStart = re.search( r'[0-9]{4}-[0-9]+-[0-9]+', start)
if searchStart:
words = searchStart.group()
term_start_chn = words
searchEnd = re.search( r'[0-9]{4}-[0-9]+-[0-9]+', end)
if searchEnd:
words = searchEnd.group()
term_end_chn = words
lines = self.get_line(self.ocr_results[pno], 'above')
searchObj = re.search( r'aboveto([0-9]+)', lines.replace('O', '0').replace('too', 'to0'))
if searchObj:
words = searchObj.group(1)
deposit_eng = f'{words}%'
lines = self.get_line(self.ocr_results[pno], '授信额度的')
searchObj = re.search( r'授信额度的([0-9]+)', lines.replace('O', '0').replace('_', ''))
if searchObj:
words = searchObj.group(1)
deposit_chn = f'{words}%'
return amount_eng, amount_chn, term_start_eng, term_end_eng, \
term_start_chn, term_end_chn, deposit_eng, deposit_chn
def get_other_arrangements_and_conditions(self):
"""获取其它约定与条件文本段落
"""
other_arrangements_and_conditions_eng = ''
other_arrangements_and_conditions_chn = ''
all_texts = ''
for pno in self.ocr_results:
for key in self.ocr_results[pno]:
all_texts += self.ocr_results[pno][key][1]
searchObj = re.search(r'Conditions:(.*?)其他', all_texts, re.I)
if searchObj:
words = searchObj.group(1)
pattern = re.compile("[\u4e00-\u9fa5]") # 去除中文字符
words = pattern.sub('', words)
other_arrangements_and_conditions_eng = words
searchObj = re.search(r'条件:(.*?)General', all_texts, re.I)
if searchObj:
words = searchObj.group(1)
other_arrangements_and_conditions_chn = words
return other_arrangements_and_conditions_eng, other_arrangements_and_conditions_chn
def get_info(self):
# 按照文档页码返回一个合同编号列表,依次表示每一页上识别到的合同编号
contract_No_list = self.get_contract_No()
self.init_result["合同编号列表"] = contract_No_list
dealer_name, dealer_No = self.get_info_in_page_3()
self.init_result["经销商名称_Page3"] = dealer_name
self.init_result["经销商统一社会信用代码或公司注册号"] = dealer_No
dealer_name = self.get_info_in_page_38()
self.init_result["经销商名称_Page38"] = dealer_name
guarantor = self.get_guarantor()
self.init_result["保证人"] = guarantor
amount_eng, amount_chn, term_start_eng, term_end_eng, \
term_start_chn, term_end_chn, deposit_eng, deposit_chn = self.get_info_in_page_39()
self.init_result["综合授信额度金额英文"] = amount_eng
self.init_result["综合授信额度金额中文"] = amount_chn
self.init_result["综合授信额度期限开始日期英文"] = term_start_eng
self.init_result["综合授信额度期限截止日期英文"] = term_end_eng
self.init_result["综合授信额度期限开始日期中文"] = term_start_chn
self.init_result["综合授信额度期限截止日期中文"] = term_end_chn
self.init_result["保证金比例英文"] = deposit_eng
self.init_result["保证金比例中文"] = deposit_chn
words_eng, words_chn = self.get_other_arrangements_and_conditions()
self.init_result["其他约定与条件英文"] = words_eng
self.init_result["其他约定与条件中文"] = words_chn
return self.init_result
class TIFFHandler:
def __init__(self, path, img_save_path):
self.path = path
self.img_save_path = img_save_path
self.img_path_list = []
def extract_image(self):
os.makedirs(self.img_save_path, exist_ok=True)
tiff = Image.open(self.path)
tiff.load()
for i in range(tiff.n_frames):
try:
save_path = os.path.join(self.img_save_path, 'page_{0}.jpeg'.format(i))
tiff.seek(i)
tiff.save(save_path)
self.img_path_list.append(save_path)
except EOFError:
break
class Command(BaseCommand, LoggerMixin):
def __init__(self):
super().__init__()
self.log_base = '[folder wsc process]'
# 处理文件开关
self.switch = True
self.sheet_name = 'Wholesales Contract'
self.finder = Finder()
# 睡眠时间
self.sleep_time = float(conf.SLEEP_SECOND_FOLDER)
# input folder
self.input_dir = conf.WSC_DIR
# ocr相关
self.go_ocr_url = conf.WSC_GO_URL
self.amount_fill = PatternFill("solid", fgColor="00FFFF00")
# 优雅退出信号：15
signal.signal(signal.SIGTERM, self.signal_handler)
def signal_handler(self, sig, frame):
self.switch = False # 停止处理文件
@staticmethod
def parse_img_path(img_path):
# 'page_{0}_img_{1}.{2}'.format(pno, img_index, ext)
img_name, _ = os.path.splitext(os.path.basename(img_path))
if re.match(r'page_\d+_img_\d+', img_name):
part_list = img_name.split('_')
return img_name, int(part_list[1])+1, int(part_list[3])+1
else:
return img_name, 1, 1
@staticmethod
def get_path(name, img_output_dir, wb_output_dir, pdf_output_dir):
time_stamp = datetime.now().strftime('%Y-%m-%d_%H_%M_%S')
new_name = '{0}_{1}'.format(time_stamp, name)
img_save_path = os.path.join(img_output_dir, new_name)
pdf_save_path = os.path.join(pdf_output_dir, new_name)
excel_name = '{0}.xlsx'.format(os.path.splitext(new_name)[0])
excel_path = os.path.join(wb_output_dir, excel_name)
return img_save_path, excel_path, pdf_save_path
@staticmethod
def get_mode_code(code_list):
result_dict = {}
for code in code_list:
if code in result_dict:
result_dict[code] += 1
else:
result_dict[code] = 1
if len(result_dict) == 1:
return None
else:
return sorted(result_dict.items(), key=lambda x:x[1], reverse=True)[0][0]
def res_process(self, all_res, excel_path):
try:
self.finder.ocr_results = all_res
results = self.finder.get_info()
wb = BSWorkbook(set(), set(), set(), set(), set())
ws = wb.create_sheet(self.sheet_name)
row_idx = 0
code_idx = 1
mode_code = None
for write_field, field_value in results.items():
row_idx += 1
if isinstance(field_value, list):
if write_field == '合同编号列表':
code_idx = row_idx
mode_code = self.get_mode_code(field_value)
ws.append((write_field, *field_value))
else:
ws.append((write_field, field_value))
if isinstance(mode_code, str):
for cell in ws[code_idx]:
if cell.value == '合同编号列表':
continue
if cell.value != mode_code:
cell.fill = self.amount_fill
wb.remove_base_sheet()
wb.save(excel_path)
except Exception as e:
self.folder_log.error('{0} [wb build error] [path={1}] [error={2}]'.format(
self.log_base, excel_path, traceback.format_exc()))
def ocr_process(self, img_path):
if os.path.exists(img_path):
# TODO 图片验证
with open(img_path, 'rb') as f:
base64_data = base64.b64encode(f.read())
# 获取解码后的base64值
file_data = base64_data.decode()
json_data = {
"file": file_data,
}
for times in range(consts.RETRY_TIMES):
try:
ocr_response = requests.post(self.go_ocr_url, json=json_data)
if ocr_response.status_code != 200:
raise OCR1Exception('{0} go status code: {1}'.format(self.log_base, ocr_response.status_code))
except Exception as e:
self.folder_log.warn('{0} [go failed] [times={1}] [img_path={2}] [error={3}]'.format(
self.log_base, times, img_path, traceback.format_exc()))
else:
ocr_res = ocr_response.json()
self.folder_log.info('{0} [ocr success] [img={1}]'.format(
self.log_base, img_path))
return ocr_res
else:
self.folder_log.warn('{0} [go failed] [img_path={1}]'.format(self.log_base, img_path))
def get_pno(self, img_path):
img_name, _ = os.path.splitext(os.path.basename(img_path))
return int(img_name.split('_')[1])
def images_process(self, img_path_list, excel_path):
all_res = {}
for img_path in img_path_list:
ocr_res = self.ocr_process(img_path)
pno = self.get_pno(img_path)
all_res[pno] = ocr_res
self.res_process(all_res, excel_path)
def pdf_process(self, name, path, img_output_dir, wb_output_dir, pdf_output_dir):
if os.path.exists(path):
try:
img_save_path, excel_path, pdf_save_path = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir)
self.folder_log.info('{0} [pdf to img start] [path={1}]'.format(self.log_base, path))
pdf_handler = PDFHandler(path, img_save_path)
pdf_handler.extract_image()
self.folder_log.info('{0} [pdf to img end] [path={1}]'.format(self.log_base, path))
except Exception as e:
self.folder_log.error('{0} [pdf to img error] [path={1}] [error={2}]'.format(
self.log_base, path, traceback.format_exc()))
raise e
else:
self.images_process(pdf_handler.img_path_list, excel_path)
shutil.move(path, pdf_save_path)
def tif_process(self, name, path, img_output_dir, wb_output_dir, tiff_output_dir):
if os.path.exists(path):
try:
img_save_path, excel_path, tiff_save_path = self.get_path(name, img_output_dir, wb_output_dir, tiff_output_dir)
self.folder_log.info('{0} [tiff to img start] [path={1}]'.format(self.log_base, path))
tiff_handler = TIFFHandler(path, img_save_path)
tiff_handler.extract_image()
self.folder_log.info('{0} [tiff to img end] [path={1}]'.format(self.log_base, path))
except Exception as e:
self.folder_log.error('{0} [tiff to img error] [path={1}] [error={2}]'.format(
self.log_base, path, traceback.format_exc()))
raise e
else:
self.images_process(tiff_handler.img_path_list, excel_path)
shutil.move(path, tiff_save_path)
def img_process(self, name, path, classify, wb_output_dir, img_output_dir, pdf_output_dir):
try:
img_save_path, excel_path, _ = self.get_path(name, img_output_dir, wb_output_dir, pdf_output_dir)
except Exception as e:
self.folder_log.error('{0} [get path error] [path={1}] [error={2}]'.format(
self.log_base, path, traceback.format_exc()))
else:
ocr_res = self.ocr_process(path, classify)
all_res = {path: ocr_res}
self.res_process(all_res, classify, excel_path)
shutil.move(path, img_save_path)
def folder_process(self, input_dir):
while not os.path.isdir(input_dir):
self.folder_log.info('{0} [input dir is not dir] [input_dir={1}]'.format(self.log_base, input_dir))
if self.switch:
time.sleep(self.sleep_time)
continue
else:
return
output_dir = os.path.join(os.path.dirname(input_dir), 'Output')
img_output_dir = os.path.join(output_dir, 'image')
wb_output_dir = os.path.join(output_dir, 'excel')
pdf_output_dir = os.path.join(output_dir, 'pdf')
tiff_output_dir = os.path.join(output_dir, 'tiff')
failed_output_dir = os.path.join(output_dir, 'failed')
os.makedirs(output_dir, exist_ok=True)
os.makedirs(img_output_dir, exist_ok=True)
os.makedirs(wb_output_dir, exist_ok=True)
os.makedirs(pdf_output_dir, exist_ok=True)
os.makedirs(tiff_output_dir, exist_ok=True)
os.makedirs(failed_output_dir, exist_ok=True)
os_error_filename_set = set()
while self.switch:
# if not os.path.isdir(input_dir):
# self.folder_log.info('{0} [input dir is not dir] [input_dir={1}]'.format(self.log_base, input_dir))
# time.sleep(self.sleep_time)
# continue
# 1. 从input dir获取pdf or image
list_dir = os.listdir(input_dir)
if not list_dir and len(os_error_filename_set) == 0:
self.folder_log.info('{0} [input dir empty] [input_dir={1}]'.format(self.log_base, input_dir))
time.sleep(self.sleep_time)
continue
all_file_set = set(list_dir)
true_file_set = all_file_set - os_error_filename_set
if len(true_file_set) == 0 and len(os_error_filename_set) > 0:
true_file_set.add(os_error_filename_set.pop())
for name in true_file_set:
time.sleep(10) # 防止文件较大时，读取到不完整文件
path = os.path.join(input_dir, name)
is_success = True
failure_reason = OfflineFailureReason.OS_ERROR.value
start_time = time.time()
try:
if not os.path.exists(path):
self.folder_log.info('{0} [path is not exists] [path={1}]'.format(self.log_base, path))
continue
elif os.path.isfile(path):
self.folder_log.info('{0} [file start] [path={1}]'.format(self.log_base, path))
if name.endswith('.pdf') or name.endswith('.PDF'):
self.pdf_process(name, path, img_output_dir, wb_output_dir, pdf_output_dir)
elif name.endswith('.tif') or name.endswith('.TIF') or name.endswith('.tiff') or \
name.endswith('.TIFF'):
self.tif_process(name, path, img_output_dir, wb_output_dir, tiff_output_dir)
else:
self.folder_log.info('{0} [path is not pdf or tif] [path={1}]'.format(
self.log_base, input_dir))
failed_path = os.path.join(failed_output_dir, '{0}_{1}'.format(time.time(), name))
shutil.move(path, failed_path)
self.folder_log.info('{0} [file end] [path={1}]'.format(self.log_base, path))
else:
self.folder_log.info('{0} [path is dir] [path={1}]'.format(self.log_base, path))
failed_path = os.path.join(failed_output_dir, '{0}_{1}'.format(time.time(), name))
shutil.move(path, failed_path)
except OSError:
is_success = False
failure_reason = OfflineFailureReason.OS_ERROR.value
os_error_filename_set.add(name)
self.folder_log.error('{0} [os error] [path={1}] [error={2}]'.format(
self.log_base, path, traceback.format_exc()))
except Exception as e:
is_success = False
failure_reason = OfflineFailureReason.PROCESS_ERROR.value
try:
self.folder_log.error('{0} [file error] [path={1}] [error={2}]'.format(self.log_base, path,
traceback.format_exc()))
failed_path = os.path.join(failed_output_dir, '{0}_{1}'.format(time.time(), name))
shutil.move(path, failed_path)
except Exception as e:
failure_reason = OfflineFailureReason.OS_ERROR.value
os_error_filename_set.add(name)
self.folder_log.error('{0} [file move error] [path={1}] [error={2}]'.format(
self.log_base, path, traceback.format_exc()))
finally:
end_time = time.time()
try:
report_table = OfflineReport if input_dir.find(consts.HIL_PREFIX) != -1 else AFCOfflineReport
report_table.objects.create(
input_folder=input_dir,
doc_type=consts.FOLDER_WSC_CLASSIFY,
file_name=name,
status=is_success,
failure_reason=failure_reason,
duration=int(end_time - start_time)
)
except Exception as e:
self.folder_log.error('{0} [db save failed] [path={1}] [error={2}]'.format(
self.log_base, path, traceback.format_exc()))
def handle(self, *args, **kwargs):
db.close_old_connections()
self.folder_process(self.input_dir)
self.folder_log.info('{0} [stop safely]'.format(self.log_base))