merge

冯轩
Showing 5 changed files with 138 additions and 0 deletions
src/apps/doc/management/commands/ocr_process.py
src/apps/doc/ocr/wb.py
src/apps/doc/views.py
src/celery_compare/tasks.py
src/common/tools/pdf_to_img.py
--- a/src/apps/doc/management/commands/ocr_process.py
View file @6e296e3
+++ b/src/apps/doc/management/commands/ocr_process.py
View file @6e296e3
--- a/src/apps/doc/ocr/wb.py
View file @6e296e3
+++ b/src/apps/doc/ocr/wb.py
View file @6e296e3
@@ -11,6 +11,8 @@ from openpyxl import Workbook
 from openpyxl.styles import PatternFill, numbers
 from openpyxl.utils import get_column_letter
 from apps.doc import consts
+import logging
+online_log = logging.getLogger('online')
 class BSWorkbook(Workbook):
@@ -562,6 +564,8 @@ class BSWorkbook(Workbook):
                borrow_cell = None if borrow_cell_idx is None or borrow_cell_idx >= length else rows[borrow_cell_idx]
                summary_cell_value = None if summary_cell is None else summary_cell.value
+                if summary_cell.value is not None:
+                    summary_cell_value = summary_cell_value.strip()
                date_cell_value = None if date_cell is None else date_cell.value
                amount_cell_value = None if amount_cell is None else amount_cell.value
                over_cell_value = None if over_cell is None else over_cell.value
@@ -638,6 +642,7 @@ class BSWorkbook(Workbook):
                # 3.2.提取信息、高亮
                # row = summary_cell.row
+                # online_log.info('[ti qu xin xi gao liang =========== >] [summary_cell_value={0}]'.format(summary_cell_value))
                if summary_cell is not None:
                    # 关键词1提取
                    if summary_cell_value in self.interest_keyword:
--- a/src/apps/doc/views.py
View file @6e296e3
+++ b/src/apps/doc/views.py
View file @6e296e3
@@ -704,6 +704,14 @@ class UploadDocView(GenericView, DocHandler):
                    classify_1 = classify_1_tmp
                    break 
+        if classify_1 == 0 and ('微信支付交易明细证明' in document_name or '微信流水' in document_name):
+            classify_1 = 12
+            self.running_log.info('[weixin bs process] [doc_id={0}]'.format(doc.id))
+        if classify_1 == 0 and (document_name.startswith("dzfp") or '电子发票' in document_name):
+            classify_1 = 29
+            self.running_log.info('[dzfp process] [doc_id={0}]'.format(doc.id))
        if document_name.endswith('.zip') or document_name.endswith('.rar') or document_name.endswith('.ZIP') \
                or document_name.endswith('.RAR'):
@@ -1258,6 +1266,14 @@ class DocView(DocGenericView, DocHandler):
                if keyword in document_name:
                    classify_1 = classify_1_tmp
                    break 
+        if classify_1 == 0 and ('微信支付交易明细证明' in document_name or '微信流水' in document_name):
+            classify_1 = 12
+            self.running_log.info('[weixin bs process] [doc_id={0}]'.format(doc.id))
+        if classify_1 == 0 and (document_name.startswith("dzfp") or '电子发票' in document_name):
+            classify_1 = 29
+            self.running_log.info('[dzfp process] [doc_id={0}]'.format(doc.id))
        # tasks = ['{0}{1}{2}'.format(prefix, consts.SPLIT_STR, doc.id)]
        task = consts.SPLIT_STR.join([prefix, str(doc.id), str(classify_1), 'N'])
--- a/src/celery_compare/tasks.py
View file @6e296e3
+++ b/src/celery_compare/tasks.py
View file @6e296e3
@@ -2731,6 +2731,13 @@ def se_compare_license(license_en, ocr_res_dict, field_list, is_auto):
                    value = json.dumps(value, ensure_ascii=False)
                error_type = empty_error_type if result == consts.RESULT_Y else ErrorType.OCR.value
                result_field_list.append((name, value, result, ocr_str, img_path, error_type, compare_logic[name][3]))
+                # CHINARPA-5620 电子发票返回CMS时 增加COMMENTS "此单为电子数控发票"
+            if ocr_field == consts.MVI_OCR_FIELD :
+                e_invoice = ocr_res_list[res_idx].get('电子发票')
+                if e_invoice is not None and e_invoice == '是':
+                    result_field_list.append(('是否电子发票', '是/否', 'Y', '是', empty_str, empty_error_type, '此单为电子数控发票'))
+                else:
+                    result_field_list.append(('是否电子发票', '是/否', 'Y', '否', empty_str, empty_error_type, '此单为电子数控发票'))
    else:
        no_ocr_result = True
@@ -3523,6 +3530,9 @@ def se_compare_process(compare_info, ocr_res_dict, is_gsyh, is_auto, id_res_list
                        elif isinstance(cn_reason, list):
                            cn_reason_list.extend(cn_reason)
                            rpa_failure_reason.setdefault('、'.join(cn_reason), []).append(value)
+                    # 
+                    if license_en == consts.MVI_EN and name == '是否电子发票' and ocr_str == '是':
+                        cn_reason_list.append(cn_reason)
                    compare_result.append(
                        {
                            consts.HEAD_LIST[0]: info_key,
--- a/src/common/tools/pdf_to_img.py
View file @6e296e3
+++ b/src/common/tools/pdf_to_img.py
View file @6e296e3
@@ -69,6 +69,7 @@ class PDFHandler:
        self.suffix = self.get_suffix(document_name)
        self.is_ebank = False
        self.is_e_pdf = False
+        self.is_e_weixin_bs = False
        self.page_text_list = []
        self.pdf_info = {}
        self.img_path_pno_list = []
@@ -186,6 +187,8 @@ class PDFHandler:
        self.img_path_list.append(img_save_path)
        if self.is_ebank:
            self.rebuild_bbox(pm.width, pm.height, page.number)
+        if self.is_e_weixin_bs:
+            self.rebuild_bbox(pm.width, pm.height, page.number)
    @staticmethod
    def getimage(pix):
@@ -407,6 +410,57 @@ class PDFHandler:
        self.is_e_pdf = True
        self.page_text_list = page_text_list
+    def put_text(self, pdf):
+        page_text_list = []
+        text_item_sum = 0
+        for pno in range(pdf.pageCount):
+            page = pdf.loadPage(pno)
+            if page.rotation is None:
+                rotation = 0
+            elif isinstance(page.rotation, int):
+                divisor, remainder = divmod(page.rotation, 90)
+                if remainder != 0:
+                    return
+                rotation = divmod(divisor, 4)[1]
+            else:
+                return
+            textpage = page.getTextPage()
+            text = textpage.extractDICT()
+            text_list = []
+            for block in text.get('blocks'):
+                for line in block.get('lines'):
+                    for span in line.get('spans'):
+                        char = span.get('text')
+                        if char.strip() == '':
+                            continue
+                        #  特殊emoji跳过
+                        try:
+                            print(char)
+                        except Exception as e:
+                            continue
+                        bbox = span.get('bbox')
+                        if pno == 0 and self.title_is_ebank(char):
+                            in_ebank_set = True
+                        text_list.append((bbox, char))
+            text_item_sum += len(text_list)
+            if text_item_sum < (pno + 1) * 5:
+                return
+            else:
+                page_text_list.append(
+                    {
+                        'width': text.get('width'),
+                        'height': text.get('height'),
+                        'rotation': rotation,
+                        'text': text_list
+                    }
+                )
+        self.is_e_pdf = True
+        self.is_e_weixin_bs = True
+        self.page_text_list = page_text_list
    def e_contract_process(self):
        os.makedirs(self.img_dir_path, exist_ok=True)
        with fitz.Document(self.path) as pdf:
@@ -473,6 +527,59 @@ class PDFHandler:
                        self.merge_il(pdf, pno, il)
        self.img_count = len(self.img_path_list)
+    def extract_image_for_weixin(self, max_img_count=None):
+        self.img_path_list = []
+        self.xref_set = set()
+        os.makedirs(self.img_dir_path, exist_ok=True)
+        if self.suffix in self.img_suffixs:
+            img_save_path = self.get_img_save_path(0, ext=self.suffix[1:])
+            shutil.copy(self.path, img_save_path)
+            self.img_path_list.append(img_save_path)
+        else:
+            with fitz.Document(self.path) as pdf:
+                # 解密
+                for pwd in self.pwd_list:
+                    if not pdf.isEncrypted:
+                        break
+                    pdf.authenticate(pwd)
+                self.metadata = pdf.metadata
+                self.page_count = pdf.pageCount
+                if isinstance(max_img_count, int) and pdf.pageCount >= max_img_count:
+                    self.img_count = pdf.pageCount
+                    return
+                self.put_text(pdf)
+                for pno in range(pdf.pageCount):
+                    il = pdf.getPageImageList(pno)  # 获取页面图片对象
+                    # (xref, smask, width, height, bpc, colorspace, alt.colorspace, name, filter, invoker)
+                    # 1.页面图片对象数目为0时，保存整个页面为png图片
+                    if self.is_e_pdf or self.is_ebank or len(il) == 0:
+                        page = pdf.loadPage(pno)
+                        self.page_to_png(page)
+                    # 2.页面图片对象数目为1时：
+                    # 小图(如电子账单的盖章)：保存整个页面为png图片
+                    # 大图：提取图片对象
+                    elif len(il) == 1:
+                        xref, smask, width, height, _, colorspace, _, _, _ = il[0]
+                        # 小图
+                        if width < WH_COUPLE_1[0] and height < WH_COUPLE_1[1]:
+                            page = pdf.loadPage(pno)
+                            self.page_to_png(page)
+                        # 大图
+                        elif width >= WH_COUPLE_6[0] or height >= WH_COUPLE_6[1]:
+                            self.is_new_modify = 1
+                            is_big_img = (width < WH_COUPLE_7[0] and height < WH_COUPLE_7[1]) # 防止图片过大
+                            page = pdf.loadPage(pno)
+                            self.page_to_png(page, is_big_img=is_big_img) 
+                        elif xref not in self.xref_set:
+                            self.extract_single_image(pdf, xref, smask, colorspace, pno)
+                    # 3.页面图片对象数目大于1时，特殊处理
+                    else:
+                        self.merge_il(pdf, pno, il)
+        self.img_count = len(self.img_path_list)
    def extract_page_image(self):
        self.img_path_list = []
        self.xref_set = set()