Skip to content

周伟奇 / bmw-ocr

Go to a project
Toggle navigation
Toggle navigation pinning
0a72fac6 by 周伟奇
Options

fix log

1 parent f5fde5ee
Inline Side-by-side
Showing 3 changed files with 58 additions and 52 deletions
src/apps/doc/management/commands/ocr_process.py
......@@ -68,10 +68,10 @@ class Command(BaseCommand, LoggerMixin):
def get_doc_info(self):
task_str, is_priority = rh.dequeue()
if task_str is None:
self.cronjob_log.info('{0} [get_doc_info] [queue empty]'.format(self.log_base))
self.online_log.info('{0} [get_doc_info] [queue empty]'.format(self.log_base))
return None, None, None
self.cronjob_log.info('{0} [get_doc_info] [task={1}] [is_priority={2}]'.format(
self.online_log.info('{0} [get_doc_info] [task={1}] [is_priority={2}]'.format(
self.log_base, task_str, is_priority))
try:
# doc, business_type = self.get_doc_object(task_str)
......@@ -81,11 +81,11 @@ class Command(BaseCommand, LoggerMixin):
doc = doc_class.objects.filter(id=doc_id).first()
if doc is None:
self.cronjob_log.warn('{0} [get_doc_info] [doc not exist] [task_str={1}] [is_priority={2}]'.format(
self.online_log.warn('{0} [get_doc_info] [doc not exist] [task_str={1}] [is_priority={2}]'.format(
self.log_base, task_str, is_priority))
return None, None, None
elif doc.status != DocStatus.INIT.value:
self.cronjob_log.warn('{0} [get_doc_info] [doc status error] [task_str={1}] [is_priority={2}] '
self.online_log.warn('{0} [get_doc_info] [doc status error] [task_str={1}] [is_priority={2}] '
'[doc_status={3}]'.format(self.log_base, task_str, is_priority, doc.status))
return None, None, None
doc.status = DocStatus.PROCESSING.value
......@@ -93,11 +93,11 @@ class Command(BaseCommand, LoggerMixin):
doc.save()
except Exception as e:
rh.enqueue([task_str], is_priority)
self.cronjob_log.error('{0} [process error (get doc info in)] [error={1}]'.format(
self.online_log.error('{0} [process error (get doc info in)] [error={1}]'.format(
self.log_base, traceback.format_exc()))
raise e
else:
self.cronjob_log.info('{0} [get_doc_info] [db save end] [task_str={1}] [is_priority={2}]'.format(
self.online_log.info('{0} [get_doc_info] [db save end] [task_str={1}] [is_priority={2}]'.format(
self.log_base, task_str, is_priority))
return doc, business_type, task_str
......@@ -107,14 +107,14 @@ class Command(BaseCommand, LoggerMixin):
# try:
# self.edms.download(pdf_path, doc.metadata_version_id)
# except Exception as e:
# self.cronjob_log.warn('{0} [edms download failed] [times={1}] [pdf_path={2}] '
# self.online_log.warn('{0} [edms download failed] [times={1}] [pdf_path={2}] '
# '[error={3}]'.format(self.log_base, times, pdf_path, traceback.format_exc()))
# edms_exc = str(e)
# else:
# break
# else:
# raise EDMSException(edms_exc)
# self.cronjob_log.info('{0} [edms download success] [pdf_path={1}]'.format(self.log_base, pdf_path))
# self.online_log.info('{0} [edms download success] [pdf_path={1}]'.format(self.log_base, pdf_path))
def bs_process(self, wb, ocr_data, bs_summary, unknown_summary, classify, res_list, pno, ino, part_idx):
sheets = ocr_data.get('data', [])
......@@ -256,7 +256,7 @@ class Command(BaseCommand, LoggerMixin):
if ocr_4_response.status_code != 200:
raise OCR4Exception('ocr_4 status code: {0}'.format(ocr_4_response.status_code))
except Exception as e:
self.cronjob_log.warn(
self.online_log.warn(
'{0} [ocr_4 failed] [times={1}] [img_path={2}] [error={3}]'.format(
self.log_base, times, img_path, traceback.format_exc()))
else:
......@@ -268,12 +268,12 @@ class Command(BaseCommand, LoggerMixin):
card_type = ocr_4_res.get('result', {}).get(
'idcard_distinguish_result', {}).get('result', -1)
self.cronjob_log.info(
self.online_log.info(
'{0} [ocr_4 success] [img_path={1}] [speed_time={2}]'.format(
self.log_base, img_path, speed_time))
break
else:
self.cronjob_log.warn(
self.online_log.warn(
'{0} [ocr_4 failed] [img_path={1}]'.format(self.log_base, img_path))
id_card_dict[consts.IC_TURE_OR_FALSE] = consts.IC_RES_MAPPING.get(card_type)
......@@ -474,7 +474,7 @@ class Command(BaseCommand, LoggerMixin):
time.sleep(self.sleep_time_doc_get)
continue
except Exception as e:
self.cronjob_log.error('{0} [process error (get doc info out)] [error={1}]'.format(
self.online_log.error('{0} [process error (get doc info out)] [error={1}]'.format(
self.log_base, traceback.format_exc()))
error_list.append(1)
return
......@@ -492,20 +492,20 @@ class Command(BaseCommand, LoggerMixin):
try:
if not doc.application_id.startswith(consts.FIXED_APPLICATION_ID_PREFIX):
self.edms.download(pdf_path, doc.metadata_version_id)
self.cronjob_log.info('{0} [edms download success] [task={1}] [times={2}] '
self.online_log.info('{0} [edms download success] [task={1}] [times={2}] '
'[pdf_path={3}]'.format(self.log_base, task_str, times, pdf_path))
# 3.PDF文件提取图片
self.cronjob_log.info('{0} [pdf to img start] [task={1}] [times={2}]'.format(
self.online_log.info('{0} [pdf to img start] [task={1}] [times={2}]'.format(
self.log_base, task_str, times))
start_time = time.time()
pdf_handler.extract_image()
end_time = time.time()
speed_time = int(end_time - start_time)
self.cronjob_log.info('{0} [pdf to img end] [task={1}] [times={2}] [spend_time={3}]'.format(
self.online_log.info('{0} [pdf to img end] [task={1}] [times={2}] [spend_time={3}]'.format(
self.log_base, task_str, times, speed_time))
except Exception as e:
self.cronjob_log.warn('{0} [download or pdf to img failed] [task={1}] [times={2}] '
self.online_log.warn('{0} [download or pdf to img failed] [task={1}] [times={2}] '
'[error={3}]'.format(self.log_base, task_str, times,
traceback.format_exc()))
else:
......@@ -515,7 +515,7 @@ class Command(BaseCommand, LoggerMixin):
img_count = len(pdf_handler.img_path_list)
if img_count == 0:
self.cronjob_log.warn('{0} [pdf to img failed (pdf img empty)] [task={1}]'.format(
self.online_log.warn('{0} [pdf to img failed (pdf img empty)] [task={1}]'.format(
self.log_base, task_str))
raise Exception('pdf img empty')
else:
......@@ -523,17 +523,17 @@ class Command(BaseCommand, LoggerMixin):
todo_count_dict[task_str] = img_count
for img_path in pdf_handler.img_path_list:
while img_queue.full():
self.cronjob_log.info('{0} [pdf_2_img_2_queue] [img queue full]'.format(self.log_base))
self.online_log.info('{0} [pdf_2_img_2_queue] [img queue full]'.format(self.log_base))
time.sleep(self.sleep_time_img_put)
img_queue.put(img_path)
# except EDMSException as e:
# try:
# doc.status = DocStatus.PROCESS_FAILED.value
# doc.save()
# self.cronjob_log.warn('{0} [process failed (edms download)] [task={1}] [error={2}]'.format(
# self.online_log.warn('{0} [process failed (edms download)] [task={1}] [error={2}]'.format(
# self.log_base, task_str, traceback.format_exc()))
# except Exception as e:
# self.cronjob_log.error('{0} [process error (db save 1)] [error={1}]'.format(
# self.online_log.error('{0} [process error (db save 1)] [error={1}]'.format(
# self.log_base, traceback.format_exc()))
# error_list.append(1)
# return
......@@ -541,10 +541,10 @@ class Command(BaseCommand, LoggerMixin):
try:
doc.status = DocStatus.PROCESS_FAILED.value
doc.save()
self.cronjob_log.warn('{0} [process failed (pdf_2_img_2_queue)] [task={1}] '
self.online_log.warn('{0} [process failed (pdf_2_img_2_queue)] [task={1}] '
'[error={2}]'.format(self.log_base, task_str, traceback.format_exc()))
except Exception as e:
self.cronjob_log.error('{0} [process error (db save)] [error={1}]'.format(
self.online_log.error('{0} [process error (db save)] [error={1}]'.format(
self.log_base, traceback.format_exc()))
error_list.append(1)
return
......@@ -554,12 +554,12 @@ class Command(BaseCommand, LoggerMixin):
try:
img_path = img_queue.get(block=False)
except Exception as e:
# self.cronjob_log.info('{0} [img_2_ocr_1] [queue empty]'.format(self.log_base))
# self.online_log.info('{0} [img_2_ocr_1] [queue empty]'.format(self.log_base))
time.sleep(self.sleep_time_img_get)
continue
else:
try:
self.cronjob_log.info('{0} [img_2_ocr_1] [get img] [img_path={1}]'.format(self.log_base, img_path))
self.online_log.info('{0} [img_2_ocr_1] [get img] [img_path={1}]'.format(self.log_base, img_path))
for times in range(consts.RETRY_TIMES):
try:
......@@ -576,23 +576,23 @@ class Command(BaseCommand, LoggerMixin):
if ocr_1_response.status_code != 200:
raise OCR1Exception('ocr_1 status code: {0}'.format(ocr_1_response.status_code))
except Exception as e:
self.cronjob_log.warn('{0} [ocr_1 failed] [times={1}] [url={2}] [img_path={3}] '
self.online_log.warn('{0} [ocr_1 failed] [times={1}] [url={2}] [img_path={3}] '
'[error={4}]'.format(self.log_base, times, url, img_path,
traceback.format_exc()))
else:
ocr_1_res = ocr_1_response.json()
end_time = time.time()
speed_time = int(end_time - start_time)
self.cronjob_log.info('{0} [ocr_1 success] [img={1}] [url={2}] [speed_time={3}]'.format(
self.online_log.info('{0} [ocr_1 success] [img={1}] [url={2}] [speed_time={3}]'.format(
self.log_base, img_path, url, speed_time))
break
else:
ocr_1_res = {}
self.cronjob_log.warn('{0} [ocr_1 failed] [img_path={1}] [url={2}]'.format(
self.online_log.warn('{0} [ocr_1 failed] [img_path={1}] [url={2}]'.format(
self.log_base, img_path, url))
# continue
except Exception as e:
self.cronjob_log.error('{0} [process error (ocr fetch)] [img_path={1}] [error={2}]'.format(
self.online_log.error('{0} [process error (ocr fetch)] [img_path={1}] [error={2}]'.format(
self.log_base, img_path, traceback.format_exc()))
else:
try:
......@@ -613,7 +613,7 @@ class Command(BaseCommand, LoggerMixin):
else:
todo_count_dict[task_str] = todo_count - 1
except Exception as e:
self.cronjob_log.error('{0} [process error (store ocr res)] [img_path={1}] [error={2}]'.format(
self.online_log.error('{0} [process error (store ocr res)] [img_path={1}] [error={2}]'.format(
self.log_base, img_path, traceback.format_exc()))
def res_2_wb(self, res_dict, img_queue, finish_queue, lock, error_list):
......@@ -621,11 +621,11 @@ class Command(BaseCommand, LoggerMixin):
try:
task_str = finish_queue.get(block=False)
except Exception as e:
# self.cronjob_log.info('{0} [res_2_wb] [queue empty]'.format(self.log_base))
# self.online_log.info('{0} [res_2_wb] [queue empty]'.format(self.log_base))
time.sleep(self.sleep_time_task_get)
continue
else:
self.cronjob_log.info('{0} [res_2_wb] [get task] [task={1}]'.format(self.log_base, task_str))
self.online_log.info('{0} [res_2_wb] [get task] [task={1}]'.format(self.log_base, task_str))
ocr_1_res = res_dict.pop(task_str, {})
business_type, doc_id_str = task_str.split(consts.SPLIT_STR)
......@@ -638,7 +638,7 @@ class Command(BaseCommand, LoggerMixin):
try:
doc = doc_class.objects.filter(id=doc_id).first()
except Exception as e:
self.cronjob_log.error('{0} [process error (db filter)] [task={1}] [error={2}]'.format(
self.online_log.error('{0} [process error (db filter)] [task={1}] [error={2}]'.format(
self.log_base, task_str, traceback.format_exc()))
else:
try:
......@@ -663,14 +663,14 @@ class Command(BaseCommand, LoggerMixin):
ocr_data_list = res.get('data', [])
if not isinstance(ocr_data_list, list):
res_list.append((pno, ino, part_idx, consts.RES_FAILED_3))
self.cronjob_log.warn('{0} [ocr_1 res error] [img={1}]'.format(self.log_base, img_path))
self.online_log.warn('{0} [ocr_1 res error] [img={1}]'.format(self.log_base, img_path))
else:
for part_idx, ocr_data in enumerate(ocr_data_list):
part_idx = part_idx + 1
classify = ocr_data.get('classify')
if classify is None:
res_list.append((pno, ino, part_idx, consts.RES_FAILED_3))
self.cronjob_log.warn('{0} [ocr_1 res error] [img={1}]'.format(
self.online_log.warn('{0} [ocr_1 res error] [img={1}]'.format(
self.log_base, img_path))
continue
elif classify in consts.OTHER_CLASSIFY_SET: # 其他类
......@@ -699,14 +699,14 @@ class Command(BaseCommand, LoggerMixin):
if ocr_2_response.status_code != 200:
raise OCR2Exception('ocr_2 status code: {0}'.format(ocr_2_response.status_code))
except Exception as e:
self.cronjob_log.warn(
self.online_log.warn(
'{0} [ocr_2 failed] [times={1}] [img_path={2}] [error={3}]'.format(
self.log_base, times, img_path, traceback.format_exc()))
else:
ocr_2_res = json.loads(ocr_2_response.text)
end_time = time.time()
speed_time = int(end_time - start_time)
self.cronjob_log.info(
self.online_log.info(
'{0} [ocr_2 success] [img={1}] [speed_time={2}]'.format(
self.log_base, img_path, speed_time))
......@@ -727,15 +727,15 @@ class Command(BaseCommand, LoggerMixin):
break
else:
res_list.append((pno, ino, part_idx, consts.RES_FAILED_2))
self.cronjob_log.warn(
self.online_log.warn(
'{0} [ocr_2 failed] [img_path={1}]'.format(self.log_base, img_path))
else: # 流水处理
self.bs_process(wb, ocr_data, bs_summary, unknown_summary, classify, res_list, pno, ino, part_idx)
else:
res_list.append((pno, ino, part_idx, consts.RES_FAILED_1))
self.cronjob_log.info('{0} [ocr_1 res error] [img={1}]'.format(self.log_base, img_path))
self.online_log.info('{0} [ocr_1 res error] [img={1}]'.format(self.log_base, img_path))
self.cronjob_log.info('{0} [task={1}] [bs_summary={2}] [unknown_summary={3}] '
self.online_log.info('{0} [task={1}] [bs_summary={2}] [unknown_summary={3}] '
'[license_summary={4}]'.format(self.log_base, task_str, bs_summary,
unknown_summary, license_summary))
......@@ -748,21 +748,21 @@ class Command(BaseCommand, LoggerMixin):
self.bs_log.info('[task={0}] [bs_summary={1}]'.format(task_str, merged_bs_summary))
self.cronjob_log.info('{0} [task={1}] [merged_bs_summary={2}] [unknown_summary={3}] '
self.online_log.info('{0} [task={1}] [merged_bs_summary={2}] [unknown_summary={3}] '
'[res_list={4}]'.format(self.log_base, task_str, merged_bs_summary,
unknown_summary, res_list))
del unknown_summary
except Exception as e:
self.cronjob_log.warn('{0} [process failed (res conformity)] [task={1}] [error={2}]'.format(
self.online_log.warn('{0} [process failed (res conformity)] [task={1}] [error={2}]'.format(
self.log_base, task_str, traceback.format_exc()))
try:
doc.status = DocStatus.PROCESS_FAILED.value
doc.save()
except Exception as e:
self.cronjob_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format(
self.online_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format(
self.log_base, task_str, traceback.format_exc()))
else:
......@@ -776,14 +776,14 @@ class Command(BaseCommand, LoggerMixin):
except Exception as e:
self.cronjob_log.warn('{0} [process failed (wb rebuild)] [task={1}] [error={2}]'.format(
self.online_log.warn('{0} [process failed (wb rebuild)] [task={1}] [error={2}]'.format(
self.log_base, task_str, traceback.format_exc()))
try:
doc.status = DocStatus.PROCESS_FAILED.value
doc.save()
except Exception as e:
self.cronjob_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format(
self.online_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format(
self.log_base, task_str, traceback.format_exc()))
else:
......@@ -793,7 +793,7 @@ class Command(BaseCommand, LoggerMixin):
try:
self.edms.upload(excel_path, doc, business_type)
except Exception as e:
self.cronjob_log.warn(
self.online_log.warn(
'{0} [edms upload failed] [times={1}] [task={2}] [error={3}]'.format(
self.log_base, times, task_str, traceback.format_exc()))
edms_exc = str(e)
......@@ -803,11 +803,11 @@ class Command(BaseCommand, LoggerMixin):
raise EDMSException(edms_exc)
except Exception as e:
doc.status = DocStatus.UPLOAD_FAILED.value
self.cronjob_log.warn('{0} [process failed (edms upload)] [task={1}] [error={2}]'.format(
self.online_log.warn('{0} [process failed (edms upload)] [task={1}] [error={2}]'.format(
self.log_base, task_str, traceback.format_exc()))
else:
doc.status = DocStatus.COMPLETE.value
self.cronjob_log.info('{0} [edms upload success] [task={1}]'.format(self.log_base, task_str))
self.online_log.info('{0} [edms upload success] [task={1}]'.format(self.log_base, task_str))
finally:
try:
doc.end_time = timezone.now()
......@@ -817,10 +817,10 @@ class Command(BaseCommand, LoggerMixin):
setattr(doc, field, count)
doc.save()
except Exception as e:
self.cronjob_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format(
self.online_log.error('{0} [process error (db save)] [task={1}] [error={2}]'.format(
self.log_base, task_str, traceback.format_exc()))
else:
self.cronjob_log.info('{0} [process complete] [task={1}]'.format(self.log_base, task_str))
self.online_log.info('{0} [process complete] [task={1}]'.format(self.log_base, task_str))
os.remove(excel_path)
finally:
pass
......@@ -837,7 +837,7 @@ class Command(BaseCommand, LoggerMixin):
pdf_path = os.path.join(doc_data_path, '{0}.pdf'.format(doc_id_str))
os.remove(pdf_path)
except Exception as e:
self.cronjob_log.error('{0} [process error (pdf & img remove)] [task={1}] [error={2}]'.format(
self.online_log.error('{0} [process error (pdf & img remove)] [task={1}] [error={2}]'.format(
self.log_base, task_str, traceback.format_exc()))
def handle(self, *args, **kwargs):
......@@ -867,4 +867,4 @@ class Command(BaseCommand, LoggerMixin):
for p in process_list:
p.join()
self.cronjob_log.info('{0} [stop safely]'.format(self.log_base))
self.online_log.info('{0} [stop safely]'.format(self.log_base))
......
src/common/mixins.py
......@@ -38,7 +38,7 @@ class GenericExceptionMixin:
class LoggerMixin:
running_log = logging.getLogger('running')
exception_log = logging.getLogger('exception')
cronjob_log = logging.getLogger('cronjob')
online_log = logging.getLogger('online')
folder_log = logging.getLogger('folder')
bs_log = logging.getLogger('bs')
license_log = logging.getLogger('license')
......
src/settings/conf/logging.conf
[loggers]
; online --> 线上OCR日志：ocr_process.py
; folder --> 线下OCR日志：folder_ocr_process.py
; bs --> 线上流水识别结果日志，用于生成每日申请号-版式报表：bs_statistics.py
; license --> 线上证件识别结果日志，用于偶尔生成结果报表：license_statistics.py
; idcard --> 线上身份证识别结果日志，用于生成每日、每月身份证报表：idcard_daily.py、idcard_monthly.py
; compare --> celery compare日志
keys=root, running, exception, online, folder, bs, license, idcard, compare, django.db.backends
[handlers]
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!