Skip to content

周伟奇 / bmw-ocr

Go to a project
Toggle navigation
Toggle navigation pinning
50ad42a7 by 周伟奇
Options

mysql --> mssql

1 parent a7e918d7
Inline Side-by-side
Showing 5 changed files with 461 additions and 22 deletions
src/apps/doc/models.py
......@@ -7,7 +7,7 @@ from .named_enum import DocStatus, KeywordsType
# 上传文件记录表/任务表
class UploadDocRecords(models.Model):
id = models.AutoField(primary_key=True, verbose_name="id")
metadata_version_id = models.CharField(max_length=64, verbose_name="元数据版本id")
metadata_version_id = models.CharField(max_length=64, verbose_name="元数据版本id") # 索引
application_id = models.CharField(max_length=64, verbose_name="申请id")
main_applicant = models.CharField(max_length=16, verbose_name="主申请人")
co_applicant = models.CharField(max_length=16, verbose_name="共同申请人")
......@@ -28,7 +28,6 @@ class UploadDocRecords(models.Model):
class HILDoc(models.Model):
id = models.AutoField(primary_key=True, verbose_name="id")
record_id = models.IntegerField(verbose_name='记录id')
metadata_version_id = models.CharField(max_length=64, verbose_name="元数据版本id")
application_id = models.CharField(max_length=64, verbose_name="申请id") # 联合索引
status = models.SmallIntegerField(default=DocStatus.INIT.value, verbose_name="文件状态") # 联合索引
......@@ -50,7 +49,6 @@ class HILDoc(models.Model):
class AFCDoc(models.Model):
id = models.AutoField(primary_key=True, verbose_name="id")
record_id = models.IntegerField(verbose_name='记录id')
metadata_version_id = models.CharField(max_length=64, verbose_name="元数据版本id")
application_id = models.CharField(max_length=64, verbose_name="申请id")
status = models.SmallIntegerField(default=DocStatus.INIT.value, verbose_name="文件状态")
......@@ -76,7 +74,7 @@ class GCAPRecords(models.Model):
status = models.IntegerField(null=True, verbose_name='状态')
rating = models.IntegerField(null=True, verbose_name='排名')
entity = models.CharField(null=True, max_length=128, verbose_name="业务类型")
application_id = models.CharField(max_length=128, verbose_name="申请id")
application_id = models.CharField(max_length=64, verbose_name="申请id")
application_version = models.IntegerField(null=True, verbose_name='申请版本')
intermediate_decision = models.CharField(null=True, max_length=128, verbose_name="中间决策")
submit_datetime = models.DateTimeField(null=True, verbose_name='提交时间')
......
src/apps/doc/views.py
......@@ -106,7 +106,7 @@ class UploadDocView(GenericView, DocHandler):
data_source = document.get('dataSource')
try:
# 1. 上传信息记录
record = UploadDocRecords.objects.create(
UploadDocRecords.objects.create(
metadata_version_id=document.get('metadataVersionId'),
application_id=application_id,
main_applicant=applicant_data.get('mainApplicantName'),
......@@ -126,7 +126,6 @@ class UploadDocView(GenericView, DocHandler):
# 2. 根据业务类型分库存储
doc_class, prefix = self.get_doc_class(business_type)
doc = doc_class.objects.create(
record_id=record.id,
metadata_version_id=document.get('metadataVersionId'),
application_id=application_id,
main_applicant=applicant_data.get('mainApplicantName'),
......@@ -142,8 +141,8 @@ class UploadDocView(GenericView, DocHandler):
is_priority = PriorityApplication.objects.filter(application_id=application_id, on_off=True).exists()
tasks = ['{0}{1}{2}'.format(prefix, consts.SPLIT_STR, doc.id)]
enqueue_res = rh.enqueue(tasks, is_priority)
self.running_log.info('[doc upload success] [args={0}] [record_id={1}] [business_type={2}] [doc_id={3}] '
'[is_priority={4}] [enqueue_res={5}]'.format(args, record.id, prefix, doc.id,
self.running_log.info('[doc upload success] [args={0}] [business_type={1}] [doc_id={2}] '
'[is_priority={3}] [enqueue_res={4}]'.format(args, prefix, doc.id,
is_priority, enqueue_res))
return response.ok()
......@@ -279,7 +278,7 @@ class DocView(GenericView, DocHandler):
document_scheme = random.choice(consts.DOC_SCHEME_LIST)
data_source = random.choice(consts.DATA_SOURCE_LIST)
business_type = random.choice(consts.BUSINESS_TYPE_LIST)
record = UploadDocRecords.objects.create(
UploadDocRecords.objects.create(
metadata_version_id=metadata_version_id,
application_id=application_id,
main_applicant='',
......@@ -296,7 +295,6 @@ class DocView(GenericView, DocHandler):
# 2. 根据业务类型分库存储
doc_class, prefix = self.get_doc_class(business_type)
doc = doc_class.objects.create(
record_id=record.id,
metadata_version_id=metadata_version_id,
application_id=application_id,
main_applicant='',
......@@ -319,7 +317,7 @@ class DocView(GenericView, DocHandler):
os.makedirs(save_dir_path, exist_ok=True)
file_write(pdf_file, save_file_path)
self.running_log.info('[mock doc upload success] [args={0}] [record_id={1}] [business_type={2}] [doc_id={3}] '
'[is_priority={4}] [enqueue_res={5}]'.format(args, record.id, prefix, doc.id,
self.running_log.info('[mock doc upload success] [args={0}] [business_type={1}] [doc_id={2}] '
'[is_priority={3}] [enqueue_res={4}]'.format(args, prefix, doc.id,
is_priority, enqueue_res))
return response.ok()
......
src/common/tools/mssql_script.py 0 → 100644
import pyodbc
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER=47.97.220.40;UID=SA;PWD=19951019Mssql', autocommit=True)
cursor = cnxn.cursor()
cursor.execute("create database afc")
cursor.execute("create database hil")
cursor.close()
cnxn.close()
hil_sql_1 = """
create table auth_group
(
id int identity
primary key,
name nvarchar(150) not null
constraint auth_group_name_a6ea08ec_uniq
unique
);
create table auth_user
(
id int identity
primary key,
password nvarchar(128) not null,
last_login datetime2,
is_superuser bit not null,
username nvarchar(150) not null
constraint auth_user_username_6821ab7c_uniq
unique,
first_name nvarchar(30) not null,
last_name nvarchar(150) not null,
email nvarchar(254) not null,
is_staff bit not null,
is_active bit not null,
date_joined datetime2 not null
);
create table auth_user_groups
(
id int identity
primary key,
user_id int not null
constraint auth_user_groups_user_id_6a12ed8b_fk_auth_user_id
references auth_user,
group_id int not null
constraint auth_user_groups_group_id_97559544_fk_auth_group_id
references auth_group
);
create unique index auth_user_groups_user_id_group_id_94350c0c_uniq
on auth_user_groups (user_id, group_id)
where [user_id] IS NOT NULL AND [group_id] IS NOT NULL;
create index auth_user_groups_group_id_97559544
on auth_user_groups (group_id);
create index auth_user_groups_user_id_6a12ed8b
on auth_user_groups (user_id);
create table django_content_type
(
id int identity
primary key,
app_label nvarchar(100) not null,
model nvarchar(100) not null
);
create table auth_permission
(
id int identity
primary key,
name nvarchar(255) not null,
content_type_id int not null
constraint auth_permission_content_type_id_2f476e4b_fk_django_content_type_id
references django_content_type,
codename nvarchar(100) not null
);
create table auth_group_permissions
(
id int identity
primary key,
group_id int not null
constraint auth_group_permissions_group_id_b120cbf9_fk_auth_group_id
references auth_group,
permission_id int not null
constraint auth_group_permissions_permission_id_84c5c92e_fk_auth_permission_id
references auth_permission
);
create index auth_group_permissions_permission_id_84c5c92e
on auth_group_permissions (permission_id);
create index auth_group_permissions_group_id_b120cbf9
on auth_group_permissions (group_id);
create unique index auth_group_permissions_group_id_permission_id_0cd325b0_uniq
on auth_group_permissions (group_id, permission_id)
where [group_id] IS NOT NULL AND [permission_id] IS NOT NULL;
create index auth_permission_content_type_id_2f476e4b
on auth_permission (content_type_id);
create unique index auth_permission_content_type_id_codename_01ab375a_uniq
on auth_permission (content_type_id, codename)
where [content_type_id] IS NOT NULL AND [codename] IS NOT NULL;
create table auth_user_user_permissions
(
id int identity
primary key,
user_id int not null
constraint auth_user_user_permissions_user_id_a95ead1b_fk_auth_user_id
references auth_user,
permission_id int not null
constraint auth_user_user_permissions_permission_id_1fbb5f2c_fk_auth_permission_id
references auth_permission
);
create index auth_user_user_permissions_permission_id_1fbb5f2c
on auth_user_user_permissions (permission_id);
create index auth_user_user_permissions_user_id_a95ead1b
on auth_user_user_permissions (user_id);
create unique index auth_user_user_permissions_user_id_permission_id_14a6b632_uniq
on auth_user_user_permissions (user_id, permission_id)
where [user_id] IS NOT NULL AND [permission_id] IS NOT NULL;
create table django_admin_log
(
id int identity
primary key,
action_time datetime2 not null,
object_id nvarchar(max),
object_repr nvarchar(200) not null,
action_flag smallint not null
constraint django_admin_log_action_flag_a8637d59_check
check ([action_flag] >= 0),
change_message nvarchar(max) not null,
content_type_id int
constraint django_admin_log_content_type_id_c4bce8eb_fk_django_content_type_id
references django_content_type,
user_id int not null
constraint django_admin_log_user_id_c564eba6_fk_auth_user_id
references auth_user
);
create index django_admin_log_content_type_id_c4bce8eb
on django_admin_log (content_type_id);
create index django_admin_log_user_id_c564eba6
on django_admin_log (user_id);
create unique index django_content_type_app_label_model_76bd3d3b_uniq
on django_content_type (app_label, model)
where [app_label] IS NOT NULL AND [model] IS NOT NULL;
create table django_migrations
(
id int identity
primary key,
app nvarchar(255) not null,
name nvarchar(255) not null,
applied datetime2 not null
);
create table django_session
(
session_key nvarchar(40) not null
primary key,
session_data nvarchar(max) not null,
expire_date datetime2 not null
);
create index django_session_expire_date_a5c62663
on django_session (expire_date);
create table oauth2_provider_application
(
id bigint identity
primary key,
client_id nvarchar(100) not null
unique,
redirect_uris nvarchar(max) not null,
client_type nvarchar(32) not null,
authorization_grant_type nvarchar(32) not null,
client_secret nvarchar(255) not null,
name nvarchar(255) not null,
user_id int
constraint oauth2_provider_application_user_id_79829054_fk_auth_user_id
references auth_user,
skip_authorization bit not null,
created datetime2 not null,
updated datetime2 not null
);
create table oauth2_provider_accesstoken
(
id bigint identity
primary key,
token nvarchar(255) not null
unique,
expires datetime2 not null,
scope nvarchar(max) not null,
application_id bigint
constraint oauth2_provider_accesstoken_application_id_b22886e1_fk_oauth2_provider_application_id
references oauth2_provider_application,
user_id int
constraint oauth2_provider_accesstoken_user_id_6e4c9a65_fk_auth_user_id
references auth_user,
created datetime2 not null,
updated datetime2 not null,
source_refresh_token_id bigint
);
create index oauth2_provider_accesstoken_user_id_6e4c9a65
on oauth2_provider_accesstoken (user_id);
create index oauth2_provider_accesstoken_application_id_b22886e1
on oauth2_provider_accesstoken (application_id);
create unique index oauth2_provider_accesstoken_source_refresh_token_id_e66fbc72_uniq
on oauth2_provider_accesstoken (source_refresh_token_id)
where [source_refresh_token_id] IS NOT NULL;
create index oauth2_provider_application_client_secret_53133678
on oauth2_provider_application (client_secret);
create index oauth2_provider_application_user_id_79829054
on oauth2_provider_application (user_id);
create table oauth2_provider_grant
(
id bigint identity
primary key,
code nvarchar(255) not null
unique,
expires datetime2 not null,
redirect_uri nvarchar(255) not null,
scope nvarchar(max) not null,
application_id bigint not null
constraint oauth2_provider_grant_application_id_81923564_fk_oauth2_provider_application_id
references oauth2_provider_application,
user_id int not null
constraint oauth2_provider_grant_user_id_e8f62af8_fk_auth_user_id
references auth_user,
created datetime2 not null,
updated datetime2 not null,
code_challenge nvarchar(128) not null,
code_challenge_method nvarchar(10) not null
);
create index oauth2_provider_grant_user_id_e8f62af8
on oauth2_provider_grant (user_id);
create index oauth2_provider_grant_application_id_81923564
on oauth2_provider_grant (application_id);
"""
hil_sql_2 = """
create table oauth2_provider_refreshtoken
(
id bigint identity
primary key,
token nvarchar(255) not null,
access_token_id bigint
constraint oauth2_provider_refreshtoken_access_token_id_775e84e8_fk_oauth2_provider_accesstoken_id
references oauth2_provider_accesstoken,
application_id bigint not null
constraint oauth2_provider_refreshtoken_application_id_2d1c311b_fk_oauth2_provider_application_id
references oauth2_provider_application,
user_id int not null
constraint oauth2_provider_refreshtoken_user_id_da837fce_fk_auth_user_id
references auth_user,
created datetime2 not null,
updated datetime2 not null,
revoked datetime2
);
alter table oauth2_provider_accesstoken
add constraint oauth2_provider_accesstoken_source_refresh_token_id_e66fbc72_fk_oauth2_provider_refreshtoken_id
foreign key (source_refresh_token_id) references oauth2_provider_refreshtoken;
create unique index oauth2_provider_refreshtoken_token_revoked_af8a5134_uniq
on oauth2_provider_refreshtoken (token, revoked)
where [token] IS NOT NULL AND [revoked] IS NOT NULL;
create index oauth2_provider_refreshtoken_user_id_da837fce
on oauth2_provider_refreshtoken (user_id);
create unique index oauth2_provider_refreshtoken_access_token_id_775e84e8_uniq
on oauth2_provider_refreshtoken (access_token_id)
where [access_token_id] IS NOT NULL;
create index oauth2_provider_refreshtoken_application_id_2d1c311b
on oauth2_provider_refreshtoken (application_id);
create table hil_doc
(
id int identity
primary key,
metadata_version_id nvarchar(64) not null,
application_id nvarchar(64) not null,
status tinyint default 0 not null,
main_applicant nvarchar(16) not null,
co_applicant nvarchar(16) not null,
guarantor_1 nvarchar(16) not null,
guarantor_2 nvarchar(16) not null,
document_name nvarchar(255) not null,
document_scheme nvarchar(64) not null,
data_source nvarchar(64) not null,
upload_finish_time datetime,
update_time datetime not null,
create_time datetime not null
);
create index hil_doc_upload_finish_time_index
on hil_doc (upload_finish_time);
create index hil_doc_create_time_index
on hil_doc (create_time);
create index hil_doc_application_id_status_index
on hil_doc (application_id, status);
create table upload_doc_records
(
id int identity
primary key,
metadata_version_id nvarchar(64) not null,
application_id nvarchar(64) not null,
main_applicant nvarchar(16) not null,
co_applicant nvarchar(16) not null,
guarantor_1 nvarchar(16) not null,
guarantor_2 nvarchar(16) not null,
document_name nvarchar(255) not null,
document_scheme nvarchar(64) not null,
business_type nvarchar(64) not null,
data_source nvarchar(64) not null,
upload_finish_time datetime,
update_time datetime not null,
create_time datetime not null
);
create unique index upload_doc_records_metadata_version_id_uindex
on upload_doc_records (metadata_version_id);
"""
afc_sql = """
create table keywords
(
id int identity primary key,
keyword nvarchar(64) not null,
type tinyint default 0 not null,
on_off bit default 1 not null,
update_time datetime not null,
create_time datetime not null
);
create table priority_application
(
id int identity
primary key,
application_id nvarchar(64) not null,
on_off bit default 1 not null,
update_time datetime not null,
create_time datetime not null
);
create index priority_application_application_id_on_off_index
on priority_application (application_id, on_off);
create table gcap_records
(
id int identity primary key,
status int,
rating int,
entity nvarchar(128),
application_id nvarchar(64) not null,
application_version int,
intermediate_decision nvarchar(128),
submit_datetime datetime,
update_time datetime not null,
create_time datetime not null
);
create table afc_doc
(
id int identity
primary key,
metadata_version_id nvarchar(64) not null,
application_id nvarchar(64) not null,
status tinyint default 0 not null,
main_applicant nvarchar(16) not null,
co_applicant nvarchar(16) not null,
guarantor_1 nvarchar(16) not null,
guarantor_2 nvarchar(16) not null,
document_name nvarchar(255) not null,
document_scheme nvarchar(64) not null,
data_source nvarchar(64) not null,
upload_finish_time datetime,
update_time datetime not null,
create_time datetime not null
);
create index afc_doc_upload_finish_time_index
on afc_doc (upload_finish_time);
create index afc_doc_create_time_index
on afc_doc (create_time);
create index afc_doc_application_id_status_index
on afc_doc (application_id, status);
"""
hil_cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER=47.97.220.40;DATABASE=hil;UID=SA;PWD=19951019Mssql', autocommit=True)
hil_cursor = hil_cnxn.cursor()
hil_cursor.execute(hil_sql_1)
hil_cursor.execute(hil_sql_2)
hil_cursor.close()
hil_cnxn.close()
afc_cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER=47.97.220.40;DATABASE=afc;UID=SA;PWD=19951019Mssql', autocommit=True)
afc_cursor = afc_cnxn.cursor()
afc_cursor.execute(afc_sql)
afc_cursor.close()
afc_cnxn.close()
src/settings/__init__.py
......@@ -95,8 +95,15 @@ DATABASES = {
'default': conf.get_namespace('MYSQL_DEFAULT_'),
'afc': conf.get_namespace('MYSQL_AFC_')
}
for db_setting in DATABASES.values():
db_setting['OPTIONS'] = {
'driver': 'ODBC Driver 17 for SQL Server'
}
DATABASE_ROUTERS = ['settings.database.DBRouter']
MYSQLPOOL_ARGUMENTS = database.MYSQLPOOL_ARGUMENTS
# MYSQLPOOL_ARGUMENTS = database.MYSQLPOOL_ARGUMENTS
# Password validation
......@@ -180,3 +187,5 @@ OAUTH2_PROVIDER = {
# this is the list of available scopes
'SCOPES': {'read': 'Read scope', 'write': 'Write scope'}
}
# print(DATABASES)
......
src/settings/database.py
......@@ -2,10 +2,10 @@
Database
https://docs.djangoproject.com/en/1.11/ref/settings/#databases
"""
import pymysql
# import pymysql
from django.db.models import options
pymysql.install_as_MySQLdb()
# pymysql.install_as_MySQLdb()
# 同一App下的Model可能会存储到不同的db，所以根据app_label来做db_router不是特别合适
# 所以添加situ_db_label来映射model到db
......@@ -13,13 +13,13 @@ pymysql.install_as_MySQLdb()
options.DEFAULT_NAMES = tuple(list(options.DEFAULT_NAMES) + ['situ_db_label'])
# 数据库连接池配置
MYSQLPOOL_ARGUMENTS = {
'recycle': 30,
'pool_size': 64,
'max_overflow': 10,
'timeout': 5,
'use_threadlocal': False,
}
# MYSQLPOOL_ARGUMENTS = {
# 'recycle': 30,
# 'pool_size': 64,
# 'max_overflow': 10,
# 'timeout': 5,
# 'use_threadlocal': False,
# }
class DBRouter(object):
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!