Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
周伟奇
/
bmw-ocr
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Graphs
Network
Create a new issue
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
b34cd942
authored
2020-11-12 17:11:04 +0800
by
周伟奇
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
add folder log
1 parent
15ebcd41
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
28 additions
and
15 deletions
src/apps/doc/management/commands/folder_ocr_process.py
src/common/mixins.py
src/settings/conf/logging.conf
src/apps/doc/management/commands/folder_ocr_process.py
View file @
b34cd94
...
...
@@ -95,7 +95,7 @@ class Command(BaseCommand, LoggerMixin):
wb
.
simple_license_rebuild
(
license_summary
,
consts
.
DOC_SCHEME_LIST
[
0
])
wb
.
save
(
excel_path
)
except
Exception
as
e
:
self
.
cronjob
_log
.
error
(
'{0} [wb build error] [path={1}] [error={2}]'
.
format
(
self
.
folder
_log
.
error
(
'{0} [wb build error] [path={1}] [error={2}]'
.
format
(
self
.
log_base
,
excel_path
,
traceback
.
format_exc
()))
def
ocr_process
(
self
,
img_path
,
classify
):
...
...
@@ -117,28 +117,28 @@ class Command(BaseCommand, LoggerMixin):
if
ocr_response
.
status_code
!=
200
:
raise
OCR1Exception
(
'{0} ocr status code: {0}'
.
format
(
self
.
log_base
,
ocr_response
.
status_code
))
except
Exception
as
e
:
self
.
cronjob
_log
.
warn
(
'{0} [ocr failed] [times={1}] [img_path={2}] [error={3}]'
.
format
(
self
.
folder
_log
.
warn
(
'{0} [ocr failed] [times={1}] [img_path={2}] [error={3}]'
.
format
(
self
.
log_base
,
times
,
img_path
,
traceback
.
format_exc
()))
else
:
ocr_res
=
ocr_response
.
json
()
end_time
=
time
.
time
()
speed_time
=
int
(
end_time
-
start_time
)
self
.
cronjob
_log
.
info
(
'{0} [ocr success] [img={1}] [res={2}] [speed_time={3}]'
.
format
(
self
.
folder
_log
.
info
(
'{0} [ocr success] [img={1}] [res={2}] [speed_time={3}]'
.
format
(
self
.
log_base
,
img_path
,
ocr_res
,
speed_time
))
return
ocr_res
else
:
self
.
cronjob
_log
.
warn
(
'{0} [ocr failed] [img_path={1}]'
.
format
(
self
.
log_base
,
img_path
))
self
.
folder
_log
.
warn
(
'{0} [ocr failed] [img_path={1}]'
.
format
(
self
.
log_base
,
img_path
))
def
pdf_process
(
self
,
name
,
path
,
classify
,
img_output_dir
,
wb_output_dir
,
pdf_output_dir
):
if
os
.
path
.
exists
(
path
):
try
:
img_save_path
,
excel_path
,
pdf_save_path
=
self
.
get_path
(
name
,
img_output_dir
,
wb_output_dir
,
pdf_output_dir
)
self
.
cronjob
_log
.
info
(
'{0} [pdf to img start] [path={1}]'
.
format
(
self
.
log_base
,
path
))
self
.
folder
_log
.
info
(
'{0} [pdf to img start] [path={1}]'
.
format
(
self
.
log_base
,
path
))
pdf_handler
=
PDFHandler
(
path
,
img_save_path
)
pdf_handler
.
extract_image
()
self
.
cronjob
_log
.
info
(
'{0} [pdf to img end] [path={1}]'
.
format
(
self
.
log_base
,
path
))
self
.
folder
_log
.
info
(
'{0} [pdf to img end] [path={1}]'
.
format
(
self
.
log_base
,
path
))
except
Exception
as
e
:
self
.
cronjob
_log
.
error
(
'{0} [pdf to img error] [path={1}] [error={2}]'
.
format
(
self
.
folder
_log
.
error
(
'{0} [pdf to img error] [path={1}] [error={2}]'
.
format
(
self
.
log_base
,
path
,
traceback
.
format_exc
()))
else
:
all_res
=
{}
...
...
@@ -155,7 +155,7 @@ class Command(BaseCommand, LoggerMixin):
try
:
img_save_path
,
excel_path
,
_
=
self
.
get_path
(
name
,
img_output_dir
,
wb_output_dir
,
pdf_output_dir
)
except
Exception
as
e
:
self
.
cronjob
_log
.
error
(
'{0} [get path error] [path={1}] [error={2}]'
.
format
(
self
.
folder
_log
.
error
(
'{0} [get path error] [path={1}] [error={2}]'
.
format
(
self
.
log_base
,
path
,
traceback
.
format_exc
()))
else
:
self
.
res_process
(
all_res
,
classify
,
excel_path
)
...
...
@@ -174,17 +174,17 @@ class Command(BaseCommand, LoggerMixin):
# 1. 从input dir获取pdf or image
list_dir
=
os
.
listdir
(
input_dir
)
if
not
list_dir
:
self
.
cronjob
_log
.
info
(
'{0} [input dir empty] [input_dir={1}]'
.
format
(
self
.
log_base
,
input_dir
))
self
.
folder
_log
.
info
(
'{0} [input dir empty] [input_dir={1}]'
.
format
(
self
.
log_base
,
input_dir
))
time
.
sleep
(
self
.
sleep_time
)
for
name
in
list_dir
:
path
=
os
.
path
.
join
(
input_dir
,
name
)
if
os
.
path
.
isfile
(
path
):
self
.
cronjob
_log
.
info
(
'{0} [file start] [path={1}]'
.
format
(
self
.
log_base
,
path
))
self
.
folder
_log
.
info
(
'{0} [file start] [path={1}]'
.
format
(
self
.
log_base
,
path
))
if
name
.
endswith
(
'.pdf'
):
self
.
pdf_process
(
name
,
path
,
classify
,
img_output_dir
,
wb_output_dir
,
pdf_output_dir
)
else
:
self
.
img_process
(
name
,
path
,
classify
,
wb_output_dir
,
img_output_dir
,
pdf_output_dir
)
self
.
cronjob
_log
.
info
(
'{0} [file end] [path={1}]'
.
format
(
self
.
log_base
,
path
))
self
.
folder
_log
.
info
(
'{0} [file end] [path={1}]'
.
format
(
self
.
log_base
,
path
))
def
handle
(
self
,
*
args
,
**
kwargs
):
process_list
=
[]
...
...
@@ -198,4 +198,4 @@ class Command(BaseCommand, LoggerMixin):
for
p
in
process_list
:
p
.
join
()
self
.
cronjob
_log
.
info
(
'{0} [stop safely]'
.
format
(
self
.
log_base
))
self
.
folder
_log
.
info
(
'{0} [stop safely]'
.
format
(
self
.
log_base
))
...
...
src/common/mixins.py
View file @
b34cd94
...
...
@@ -39,6 +39,7 @@ class LoggerMixin:
running_log
=
logging
.
getLogger
(
'running'
)
exception_log
=
logging
.
getLogger
(
'exception'
)
cronjob_log
=
logging
.
getLogger
(
'cronjob'
)
folder_log
=
logging
.
getLogger
(
'folder'
)
class
GenericView
(
LoggerMixin
,
GenericExceptionMixin
,
GenericAPIView
):
...
...
src/settings/conf/logging.conf
View file @
b34cd94
[
loggers
]
keys
=
root
,
running
,
exception
,
cronjob
,
django
.
db
.
backends
keys
=
root
,
running
,
exception
,
cronjob
,
folder
,
django
.
db
.
backends
[
handlers
]
keys
=
consoleHandler
,
django_rotateFileHandler
,
exceptionFileHandler
,
cronjobFileHandler
,
djangodbFileHandler
keys
=
consoleHandler
,
django_rotateFileHandler
,
exceptionFileHandler
,
cronjobFileHandler
,
folderFileHandler
,
djangodbFileHandler
[
formatters
]
keys
=
SituFormatter
,
dataLogFormatter
...
...
@@ -37,7 +37,13 @@ args=('../logs/exception.log',)
class
=
situlogger
.
SituRotatingFileHandler
level
=
DEBUG
formatter
=
SituFormatter
args
=(
'../logs/cronjob.log'
,)
args
=(
'../logs/online_ocr.log'
,)
[
handler_folderFileHandler
]
class
=
situlogger
.
SituRotatingFileHandler
level
=
DEBUG
formatter
=
SituFormatter
args
=(
'../logs/folder_ocr.log'
,)
[
handler_djangodbFileHandler
]
class
=
situlogger
.
SituRotatingFileHandler
...
...
@@ -67,6 +73,12 @@ handlers=cronjobFileHandler
qualname
=
cronjob
propagate
=
0
[
logger_folder
]
level
=
INFO
handlers
=
folderFileHandler
qualname
=
folder
propagate
=
0
[
logger_django
.
db
.
backends
]
level
=
DEBUG
handlers
=
djangodbFileHandler
...
...
Write
Preview
Styling with
Markdown
is supported
Attach a file
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to post a comment